diff --git a/.azure-pipelines/esrp/sign.yml b/.azure-pipelines/esrp/sign.yml new file mode 100644 index 00000000000000..b4d14d2713ee8f --- /dev/null +++ b/.azure-pipelines/esrp/sign.yml @@ -0,0 +1,106 @@ +# Reusable step template for ESRP code signing via EsrpCodeSigning@6. +# +# For macOS, ESRP requires files to be submitted as a zip archive. +# Set 'useArchive: true' to automatically handle the +# copy → zip → sign → extract cycle. For Windows/Linux where ESRP +# can sign files directly in a folder, leave it as false (default). +# +parameters: + - name: displayName + type: string + - name: folderPath + type: string + - name: pattern + type: string + - name: inlineOperation + type: string + # When true, matching files are copied to a staging dir, zipped, + # signed, and extracted back to folderPath. + - name: useArchive + type: boolean + default: false + # ESRP connection parameters (defaults use pipeline variables) + - name: connectedServiceName + type: string + default: $(esrpAppConnectionName) + - name: appRegistrationClientId + type: string + default: $(esrpClientId) + - name: appRegistrationTenantId + type: string + default: $(esrpTenantId) + - name: authAkvName + type: string + default: $(esrpKeyVaultName) + - name: authSignCertName + type: string + default: $(esrpSignReqCertName) + - name: serviceEndpointUrl + type: string + default: $(esrpEndpointUrl) + +steps: + - ${{ if eq(parameters.useArchive, true) }}: + - task: DeleteFiles@1 + displayName: 'Clean staging dir for ${{ parameters.displayName }}' + inputs: + SourceFolder: '$(Agent.TempDirectory)/esrp-staging' + Contents: '*' + RemoveSourceFolder: true + - task: CopyFiles@2 + displayName: 'Collect files for ${{ parameters.displayName }}' + inputs: + SourceFolder: '${{ parameters.folderPath }}' + Contents: '${{ parameters.pattern }}' + TargetFolder: '$(Agent.TempDirectory)/esrp-staging/contents' + - task: ArchiveFiles@2 + displayName: 'Archive files for ${{ parameters.displayName }}' + inputs: + rootFolderOrFile: '$(Agent.TempDirectory)/esrp-staging/contents' + includeRootFolder: false + archiveType: zip + archiveFile: '$(Agent.TempDirectory)/esrp-staging/archive.zip' + - task: EsrpCodeSigning@6 + displayName: '${{ parameters.displayName }}' + inputs: + connectedServiceName: '${{ parameters.connectedServiceName }}' + useMSIAuthentication: true + appRegistrationClientId: '${{ parameters.appRegistrationClientId }}' + appRegistrationTenantId: '${{ parameters.appRegistrationTenantId }}' + authAkvName: '${{ parameters.authAkvName }}' + authSignCertName: '${{ parameters.authSignCertName }}' + serviceEndpointUrl: '${{ parameters.serviceEndpointUrl }}' + folderPath: '$(Agent.TempDirectory)/esrp-staging' + pattern: 'archive.zip' + useMinimatch: true + signConfigType: inlineSignParams + inlineOperation: ${{ parameters.inlineOperation }} + - task: ExtractFiles@1 + displayName: 'Extract signed files for ${{ parameters.displayName }}' + inputs: + archiveFilePatterns: '$(Agent.TempDirectory)/esrp-staging/archive.zip' + destinationFolder: '${{ parameters.folderPath }}' + overwriteExistingFiles: true + - task: DeleteFiles@1 + displayName: 'Clean up staging dir for ${{ parameters.displayName }}' + condition: always() + inputs: + SourceFolder: '$(Agent.TempDirectory)/esrp-staging' + Contents: '*' + RemoveSourceFolder: true + - ${{ else }}: + - task: EsrpCodeSigning@6 + displayName: '${{ parameters.displayName }}' + inputs: + connectedServiceName: '${{ parameters.connectedServiceName }}' + useMSIAuthentication: true + appRegistrationClientId: '${{ parameters.appRegistrationClientId }}' + appRegistrationTenantId: '${{ parameters.appRegistrationTenantId }}' + authAkvName: '${{ parameters.authAkvName }}' + authSignCertName: '${{ parameters.authSignCertName }}' + serviceEndpointUrl: '${{ parameters.serviceEndpointUrl }}' + folderPath: '${{ parameters.folderPath }}' + pattern: '${{ parameters.pattern }}' + useMinimatch: true + signConfigType: inlineSignParams + inlineOperation: ${{ parameters.inlineOperation }} diff --git a/.azure-pipelines/esrp/windows/esrpsign.sh b/.azure-pipelines/esrp/windows/esrpsign.sh new file mode 100755 index 00000000000000..ee2ed2db5d9e2d --- /dev/null +++ b/.azure-pipelines/esrp/windows/esrpsign.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# +# Sign Windows files using the ESRP client (Authenticode). +# Usage: esrpsign.sh [file2 ...] +# +# Required environment variables: +# ESRP_TOOL - Path to ESRPClient.exe +# ESRP_AUTH - Path to the ESRP auth JSON file +# SYSTEM_ACCESSTOKEN - ADO system access token (OAuth bearer) +# +# Optional environment variables: +# ESRP_KEYCODE - Signing key code (default: CP-231522) +# +# The script generates the auth and input JSON files and sets the +# following ESRP client environment variables automatically: +# ESRP_AUTH_CONFIG - Path to the auth JSON file +# ESRP_POLICY_CONFIG - Path to the policy JSON file +# ESRP_SESSION_CONFIG - Not set; ESRP client defaults are used +# +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: esrpsign.sh [file ...]" >&2 + exit 1 +fi + +if [ -z "${ESRP_TOOL:-}" ]; then + echo "error: ESRP_TOOL environment variable must be set" >&2 + exit 1 +fi +if [ -z "${ESRP_AUTH:-}" ]; then + echo "error: ESRP_AUTH environment variable must be set" >&2 + exit 1 +fi +if [ -z "${SYSTEM_ACCESSTOKEN:-}" ]; then + echo "error: SYSTEM_ACCESSTOKEN environment variable must be set" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/../../scripts/windows/utils.sh" + +# Check for overriden key code, otherwise use default (Microsoft Third-Party/OSS) +ESRP_KEYCODE="${ESRP_KEYCODE:-CP-231522}" + +# Create work dir and resolve its Windows path by cd-ing into it. +WORK_DIR="$(mktemp -d)" +WORK_DIR_WIN="$(cd "$WORK_DIR" && pwd -W | sed 's|/|\\|g')" + +echo "==> ESRP signing tool: $ESRP_TOOL" +echo "==> Working directory: $WORK_DIR" + +if [ ! -f "$ESRP_TOOL" ]; then + echo "error: ESRPClient.exe not found at $ESRP_TOOL" >&2 + exit 1 +fi + +# Build the SignRequestFiles JSON array +echo "==> Preparing files for signing ($# file(s))..." +files_json="" +for file in "$@"; do + if [ ! -f "$file" ]; then + echo "error: file not found: $file" >&2 + exit 1 + fi + + abs_path="$(cd "$(dirname "$file")" && pwd)/$(basename "$file")" + win_path="$(to_windows_path "$abs_path")" + # Escape backslashes for JSON + win_path_escaped="${win_path//\\/\\\\}" + echo " - $win_path" + + if [ -n "$files_json" ]; then + files_json+="," + fi + files_json+=" + { + \"SourceLocation\": \"$win_path_escaped\", + \"DestinationLocation\": \"$win_path_escaped\" + }" +done + +# Generate the input JSON +input_json="$WORK_DIR/input.json" +output_json="$WORK_DIR/output.json" + +echo "==> Generating input JSON: $input_json" +cat > "$input_json" <<-EOF + { + "Version": "1.0.0", + "SignBatches": [ + { + "SourceLocationType": "UNC", + "DestinationLocationType": "UNC", + "SignRequestFiles": [$files_json + ], + "SigningInfo": { + "Operations": [ + { + "KeyCode": "$ESRP_KEYCODE", + "OperationCode": "SigntoolSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "OpusName": "Microsoft", + "OpusInfo": "https://www.microsoft.com", + "FileDigest": "/fd SHA256", + "PageHash": "/NPH", + "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + }, + { + "KeyCode": "$ESRP_KEYCODE", + "OperationCode": "SigntoolVerify", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": {} + } + ] + } + } + ] + } +EOF + +# Generate policy JSON +echo "==> Generating policy JSON..." +policy_json="$WORK_DIR/policy.json" +cat > "$policy_json" <<-EOF + { + "Version": "1.0.0", + "Intent": "ProductRelease", + "ContentType": "Binaries", + "ContentOrigin": "1stParty", + "ProductState": "Current", + "Audience": "ExternalBroad" + } +EOF + +# Use auth JSON from ESRP_AUTH +export ESRP_AUTH_CONFIG="$(to_windows_path "$ESRP_AUTH")" +export ESRP_POLICY_CONFIG="$WORK_DIR_WIN\\policy.json" + +# The ADO system access token is referenced in the auth JSON via the environment +# variable - export this so the ESRP client can pick it up when it runs. +export SYSTEM_ACCESSTOKEN + +# Print generated JSON files for debugging +echo "==> Auth JSON:" +cat "$ESRP_AUTH" +echo "" +echo "==> Policy JSON:" +cat "$policy_json" +echo "" +echo "==> Input JSON:" +cat "$input_json" +echo "" + +# Sign the files +esrp_tool_win="$(to_windows_path "$ESRP_TOOL")" +input_json_win="$WORK_DIR_WIN\\input.json" +output_json_win="$WORK_DIR_WIN\\output.json" + +echo "==> ESRP_AUTH_CONFIG=$ESRP_AUTH_CONFIG" +echo "==> ESRP_POLICY_CONFIG=$ESRP_POLICY_CONFIG" +echo "==> Running: $esrp_tool_win sign -i $input_json_win -o $output_json_win" +"$esrp_tool_win" sign \ + -i "$input_json_win" \ + -o "$output_json_win" + +echo "==> Signing complete." +echo "==> Output JSON:" +cat "$output_json" diff --git a/.azure-pipelines/esrp/windows/setup.yml b/.azure-pipelines/esrp/windows/setup.yml new file mode 100644 index 00000000000000..c7eb655c1586c2 --- /dev/null +++ b/.azure-pipelines/esrp/windows/setup.yml @@ -0,0 +1,69 @@ +parameters: + - name: serviceConnectionName + type: string + - name: esrpClientId + type: string + - name: keyVaultName + type: string + - name: signCertName + type: string + +steps: + - task: EsrpClientTool@5 + name: esrpinstall + displayName: 'Install ESRP client' + - task: AzureCLI@2 + displayName: 'Set up ESRP environment' + inputs: + azureSubscription: ${{ parameters.serviceConnectionName }} + addSpnToEnvironment: true + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Resolve ESRP client tool path (passed via env to avoid PS subexpression issues) + $esrpTool = "$env:ESRPCLIENT_TOOLPATH\$env:ESRPCLIENT_TOOLNAME" + if (-not (Test-Path $esrpTool)) { Write-Error "ESRPClient.exe not found at $esrpTool"; exit 1 } + Write-Host "Found ESRP client: $esrpTool" + Write-Host "##vso[task.setvariable variable=ESRP_TOOL]$esrpTool" + + # Derive the service connection GUID from the ENDPOINT_URL_* env vars + # that the agent emits for the bound connection. Filter out the + # built-in SystemVssConnection which is always present. + $scId = (Get-ChildItem env:ENDPOINT_URL_*).Name ` + -replace '^ENDPOINT_URL_','' | + Where-Object { $_ -ne 'SYSTEMVSSCONNECTION' } + if (-not $scId) { Write-Error "Could not derive service connection GUID"; exit 1 } + Write-Host "Resolved service connection GUID: $scId" + + # servicePrincipalId and tenantId are provided by addSpnToEnvironment + $authJson = @{ + Version = "1.0.0" + AuthenticationType = "AAD_MSI_WIF" + EsrpClientId = "${{ parameters.esrpClientId }}" + ClientId = $env:servicePrincipalId + TenantId = $env:tenantId + AADAuthorityBaseUri = "https://login.microsoftonline.com/" + FederatedTokenData = @{ + JobId = "$(System.JobId)" + PlanId = "$(System.PlanId)" + ProjectId = "$(System.TeamProjectId)" + Hub = "$(System.HostType)" + Uri = "$(System.CollectionUri)" + ServiceConnectionId = $scId + SystemAccessToken = "SYSTEM_ACCESSTOKEN" + } + RequestSigningCert = @{ + GetCertFromKeyVault = $true + KeyVaultName = "${{ parameters.keyVaultName }}" + KeyVaultCertName = "${{ parameters.signCertName }}" + } + } | ConvertTo-Json -Depth 4 + + $authPath = "$(Agent.TempDirectory)\esrp-auth.json" + $authJson | Set-Content -Path $authPath -Encoding UTF8 + Write-Host "Generated ESRP auth JSON: $authPath" + Write-Host "##vso[task.setvariable variable=ESRP_AUTH]$authPath" + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + ESRPCLIENT_TOOLPATH: $(esrpinstall.esrpclient.toolpath) + ESRPCLIENT_TOOLNAME: $(esrpinstall.esrpclient.toolname) diff --git a/.azure-pipelines/patches/.gitattributes b/.azure-pipelines/patches/.gitattributes new file mode 100644 index 00000000000000..ef9170ec0077a5 --- /dev/null +++ b/.azure-pipelines/patches/.gitattributes @@ -0,0 +1 @@ +*.patch whitespace=-trailing-space,-blank-at-eof diff --git a/.azure-pipelines/patches/windows/build-extra/0000-installer-publisher.patch b/.azure-pipelines/patches/windows/build-extra/0000-installer-publisher.patch new file mode 100644 index 00000000000000..aaef8b8a35b6d1 --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0000-installer-publisher.patch @@ -0,0 +1,13 @@ +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..137f660 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -65,7 +65,7 @@ SignTool=signtool + ; Installer-related + AllowNoIcons=yes + AppName={#APP_NAME} +-AppPublisher=The Git Development Community ++AppPublisher=The Git Client Team at Microsoft + AppPublisherURL={#APP_URL} + AppSupportURL={#APP_CONTACT_URL} + AppVersion={#APP_VERSION} diff --git a/.azure-pipelines/patches/windows/build-extra/0001-installer-vsintegration.patch b/.azure-pipelines/patches/windows/build-extra/0001-installer-vsintegration.patch new file mode 100644 index 00000000000000..6797e6ab6c88ba --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0001-installer-vsintegration.patch @@ -0,0 +1,50 @@ +diff --git a/installer/helpers.inc.iss b/installer/helpers.inc.iss +index 3e3788d..fc81be7 100644 +--- a/installer/helpers.inc.iss ++++ b/installer/helpers.inc.iss +@@ -224,3 +224,25 @@ begin + DeleteFile(OutPath); + DeleteFile(ErrPath); + end; ++ ++procedure CustomPostInstall(); ++begin ++ if not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) then ++ LogError('Could not register TeamFoundation\GitSourceControl'); ++end; ++ ++procedure CustomPostUninstall(); ++begin ++ if not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath') then ++ LogError('Could not register TeamFoundation\GitSourceControl'); ++end; +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..74d8375 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -3603,6 +3603,7 @@ begin + Install a scheduled task to try to auto-update Git for Windows + } + ++ CustomPostInstall(); + if IsComponentInstalled('autoupdate') then begin + WizardForm.StatusLabel.Caption:='Set up daily up to date check'; + InstallAutoUpdater(); +@@ -3943,6 +3944,7 @@ begin + Remove the scheduled task to try to auto-update Git for Windows + } + ++ CustomPostUninstall(); + if IsComponentInstalled('autoupdate') then + UninstallAutoUpdater(); + diff --git a/.azure-pipelines/patches/windows/build-extra/0002-installer-default-components.patch b/.azure-pipelines/patches/windows/build-extra/0002-installer-default-components.patch new file mode 100644 index 00000000000000..2135ecf925bc84 --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0002-installer-default-components.patch @@ -0,0 +1,17 @@ +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..37d79b0 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -1925,6 +1925,12 @@ begin + GetDefaultsFromGitConfig('system'); + + ChosenOptions:=''; ++ if (ExpandConstant('{param:components|/}')='/') then begin ++ WizardSelectComponents('autoupdate'); ++#ifdef WITH_SCALAR ++ WizardSelectComponents('scalar'); ++#endif ++ end; + + PrevPageID:=wpSelectProgramGroup; + diff --git a/.azure-pipelines/patches/windows/build-extra/0003-installer-fork-from-microsoft-git.patch b/.azure-pipelines/patches/windows/build-extra/0003-installer-fork-from-microsoft-git.patch new file mode 100644 index 00000000000000..28af996716dc03 --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0003-installer-fork-from-microsoft-git.patch @@ -0,0 +1,29 @@ +diff --git a/git-update-git-for-windows.config b/git-update-git-for-windows.config +new file mode 100644 +index 0000000..bfd0744 +--- /dev/null ++++ b/git-update-git-for-windows.config +@@ -0,0 +1,2 @@ ++[update] ++ fromFork = microsoft/git +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..71d5e72 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -126,6 +126,7 @@ Filename: {app}\ReleaseNotes.html; Description: View Release Notes; Flags: shell + [Files] + ; Install files that might be in use during setup under a different name. + #include "file-list.iss" ++Source: {#SourcePath}\..\git-update-git-for-windows.config; DestDir: {app}\{#MINGW_BITNESS}\bin; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore + Source: {#SourcePath}\ReleaseNotes.html; DestDir: {app}; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore + Source: {#SourcePath}\..\LICENSE.txt; DestDir: {app}; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore + Source: {#SourcePath}\NOTICE.txt; DestDir: {app}; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore; Check: ParamIsSet('VSNOTICE') +@@ -275,6 +276,8 @@ Type: files; Name: {app}\etc\rebase.db.i386 + Type: files; Name: {app}\etc\install-options.txt + Type: dirifempty; Name: {app}\{#MINGW_BITNESS}\libexec\git-core + Type: dirifempty; Name: {app}\{#MINGW_BITNESS}\libexec ++Type: files; Name: {app}\{#MINGW_BITNESS}\bin\git-update-git-for-windows.config ++Type: dirifempty; Name: {app}\{#MINGW_BITNESS}\bin + Type: dirifempty; Name: {app}\{#MINGW_BITNESS} + Type: dirifempty; Name: {app} + diff --git a/.azure-pipelines/patches/windows/git-sdk/0000-update-recently-seen.patch b/.azure-pipelines/patches/windows/git-sdk/0000-update-recently-seen.patch new file mode 100644 index 00000000000000..cf73dfd5f3f449 --- /dev/null +++ b/.azure-pipelines/patches/windows/git-sdk/0000-update-recently-seen.patch @@ -0,0 +1,12 @@ +diff --git a/bin/git-update-git-for-windows b/bin/git-update-git-for-windows +index 29444d9..6705da1 100644 +--- a/bin/git-update-git-for-windows ++++ b/bin/git-update-git-for-windows +@@ -4,6 +4,7 @@ + # release. If versions differ, the bit matched installer is downloaded and run + # when confirmation to do so is given. + ++use_recently_seen=no + + # Compare version strings + # Prints -1, 0 or 1 to stdout diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml new file mode 100644 index 00000000000000..076b663e884e77 --- /dev/null +++ b/.azure-pipelines/release.yml @@ -0,0 +1,1272 @@ +name: $(Date:yyyyMMdd)$(Rev:.r) +trigger: + branches: + exclude: + - '*' + tags: + include: + - v[0-9]*vfs* +pr: none + +resources: + repositories: + - repository: 1ESPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +parameters: + - name: 'esrp' + type: boolean + default: true + displayName: 'Enable ESRP code signing' + - name: 'github' + type: boolean + default: true + displayName: 'Enable GitHub release publishing' + - name: 'versionOverride' + type: string + default: '-' + displayName: 'Version override (release publishing is skipped if set)' + +# +# 1ES Pipeline Templates do not allow using a matrix strategy so we create +# a YAML object parameter with and foreach to create jobs for each entry. +# Each OS has its own matrix object since their build steps differ. +# + - name: windows_matrix + type: object + default: + - id: windows_x64 + jobName: 'Windows (x64)' + pool: GitClientPME-1ESHostedPool-intel-pc + poolArch: amd64 + image: win-x86_64-ado1es + os: windows + toolchain: ucrt-x86_64 + cpu_arch: x86_64 + mingwprefix: ucrt64 + msystem: UCRT64 + sdk_repo: git-for-windows/git-sdk-64 + + - id: windows_arm64 + jobName: 'Windows (ARM64)' + pool: GitClientPME-1ESHostedPool-arm64-pc + poolArch: arm64 + image: win-arm64-ado1es + os: windows + toolchain: clang-aarch64 + cpu_arch: aarch64 + mingwprefix: clangarm64 + msystem: CLANGARM64 + sdk_repo: git-for-windows/git-sdk-arm64 + + - name: macos_matrix + type: object + default: + - id: macos_arm64 + jobName: 'macOS (ARM64)' + pool: 'Azure Pipelines' + image: macOS-15-arm64 + os: macos + + - name: linux_matrix + type: object + default: + - id: linux_x64 + jobName: 'Linux (x64)' + pool: GitClientPME-1ESHostedPool-intel-pc + poolArch: amd64 + image: ubuntu-x86_64-ado1es + os: linux + cc_arch: x86_64 + deb_arch: amd64 + + - id: linux_arm64 + jobName: 'Linux (ARM64)' + pool: GitClientPME-1ESHostedPool-arm64-pc + poolArch: arm64 + image: ubuntu-arm64-ado1es + os: linux + cc_arch: aarch64 + deb_arch: arm64 + +variables: + - name: 'esrpAppConnectionName' + value: '1ESGitClient-ESRP-App' + - name: 'githubConnectionName' + value: 'GitHub-MicrosoftGit' + # ESRP signing variables set in the pipeline settings: + # - esrpEndpointUrl + # - esrpMI + # - esrpClientId + # - esrpTenantId + # - esrpKeyVaultName + # - esrpSignReqCertName + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelines + parameters: + sdl: + # SDL source analysis tasks only run on Windows images + sourceAnalysisPool: + name: GitClientPME-1ESHostedPool-intel-pc + image: win-x86_64-ado1es + os: windows + stages: + - stage: prereqs + displayName: 'Prerequisites' + jobs: + - job: prebuild + displayName: 'Pre-build validation' + pool: + name: GitClientPME-1ESHostedPool-intel-pc + image: ubuntu-x86_64-ado1es + os: linux + steps: + - checkout: self + fetchDepth: 0 + fetchTags: true + - ${{ if or(eq(parameters.versionOverride, ''), eq(parameters.versionOverride, '-')) }}: + - task: Bash@3 + displayName: 'Resolve version and tag information' + name: info + inputs: + targetType: filePath + filePath: .azure-pipelines/scripts/resolve-version.sh + - ${{ if and(ne(parameters.versionOverride, ''), ne(parameters.versionOverride, '-')) }}: + - task: Bash@3 + displayName: 'Set version override information' + name: info + inputs: + targetType: inline + script: | + tag_sha=$(git rev-parse HEAD) + echo "##vso[task.logissue type=warning]Using version override: ${{ parameters.versionOverride }}. Release publishing will be skipped." + echo "Git version: ${{ parameters.versionOverride }}" + echo "Tag name: untagged" + echo "Tag SHA: ${tag_sha}" + echo "##vso[task.setvariable variable=git_version;isOutput=true;isReadOnly=true]${{ parameters.versionOverride }}" + echo "##vso[task.setvariable variable=tag_name;isOutput=true;isReadOnly=true]untagged" + echo "##vso[task.setvariable variable=tag_sha;isOutput=true;isReadOnly=true]${tag_sha}" + echo "##vso[build.updatebuildnumber][UNTAGGED] ${tag_sha} (${BUILD_BUILDNUMBER:-unknown})" + + - stage: build + displayName: 'Build' + dependsOn: [prereqs] + jobs: + # + # Windows build jobs + # + - ${{ each dim in parameters.windows_matrix }}: + - job: ${{ dim.id }} + displayName: ${{ dim.jobName }} + pool: + name: ${{ dim.pool }} + image: ${{ dim.image }} + os: ${{ dim.os }} + hostArchitecture: ${{ dim.poolArch }} + variables: + tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] + tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] + git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] + toolchain: ${{ dim.toolchain }} + mingwprefix: ${{ dim.mingwprefix }} + sdk_repo: ${{ dim.sdk_repo }} + cpu_arch: ${{ dim.cpu_arch }} + templateContext: + sdl: + suppression: + suppressionFile: $(Build.SourcesDirectory)/.azure-pipelines/sdl/${{ dim.id }}/.gdnsuppress + binskim: + # Direct binskim to analyze the built product binaries rather + # than the installer/7z outputs. Binskim cannot crack open the + # installer or 7z archive to find the binaries inside, and + # these outputs are generated by external tools (not possible + # to resolve any warnings about them). + # + # The 'Extract mingw-w64-git packages for binary analysis' + # step below stages only the first-party pacman packages + # produced by `please.sh build-mingw-w64-git` + # (mingw-w64--{git,git-credential-wincred, + # git-pdb}-*.pkg.tar.xz) into _bin//. By + # construction, that tree contains only binaries built + # from this repo's Makefile (git.exe, the dashed + # subcommands, scalar.exe, headless-git.exe, + # git-gvfs-helper.exe, git-credential-wincred.exe, ...) + # plus their cv2pdb-generated .pdbs, so a broad **/*.{exe, + # dll} glob is safe. This excludes the third-party + # payload carried by the full Git for Windows installer: + # MSYS2/MinGW runtime, Perl, Tcl/Tk, libcurl/libssl/libssh2, + # Git Credential Manager, Git LFS, tig, and the + # build-extra git-wrapper launcher shims. + enabled: true + analyzeTargetGlob: '$(Build.ArtifactStagingDirectory)/_bin/${{ dim.mingwprefix }}/**/*.exe;$(Build.ArtifactStagingDirectory)/_bin/${{ dim.mingwprefix }}/**/*.dll' + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/_final' + artifactName: '${{ dim.id }}' + steps: + - checkout: self + # Add Git Bash to the PATH so Bash tasks can find it + - task: BatchScript@1 + displayName: 'Add Git Bash to PATH' + inputs: + filename: ./.azure-pipelines/scripts/windows/setup-git-bash.cmd + # Install VS 2022 Build Tools on x64 so cv2pdb-strip can locate + # mspdb140.dll. We do not need to do this on ARM64 since we use + # clang's llvm-strip there instead. + - ${{ if eq(dim.poolArch, 'amd64') }}: + - task: PowerShell@2 + displayName: 'Setup cv2pdb (x64)' + inputs: + filePath: ./.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 + - task: Bash@3 + displayName: 'Install Git for Windows SDK' + inputs: + filePath: ./.azure-pipelines/scripts/windows/setup-git-sdk.sh + arguments: '$(sdk_repo) $(mingwprefix) "$(Agent.TempDirectory)\gitsdk"' + env: + BOOTSTRAP_DIR: '$(Build.SourcesDirectory)' + # please.sh's `create-sdk-artifact` step does the + # final sparse-checkout of the build-installers + # SDK subset, which is I/O-bound (lots of small + # writes), not CPU-bound. The default + # checkout.workers=1 leaves the agent's I/O + # subsystem mostly idle; bumping it well beyond + # the core count gives a substantial speedup. + GIT_CONFIG_PARAMETERS: "'checkout.workers=56'" + - task: Bash@3 + displayName: 'Clone build-extra into SDK' + inputs: + targetType: inline + script: | + set -euo pipefail + # The please.sh + signtool.sh scripts the build + # relies on live in build-extra; the SDK ships + # without them. Partial clone to keep this fast. + git clone --filter=blob:none --single-branch -b main \ + https://github.com/git-for-windows/build-extra \ + /usr/src/build-extra + # Setup ESRP code signing for Windows (sets ESRP_TOOL, + # ESRP_AUTH) before the build steps so that the build + # itself (Inno Setup, makepkg-mingw) can invoke ESRP + # via the `git signtool` alias for in-line signing of + # individual binaries. + - ${{ if eq(parameters.esrp, true) }}: + - template: .azure-pipelines/esrp/windows/setup.yml@self + parameters: + serviceConnectionName: $(esrpAppConnectionName) + esrpClientId: $(esrpClientId) + keyVaultName: $(esrpKeyVaultName) + signCertName: $(esrpSignReqCertName) + - task: Bash@3 + displayName: 'Configure git signtool alias for ESRP' + inputs: + targetType: inline + script: | + set -euo pipefail + # please.sh, makepkg-mingw, and Inno Setup's + # release.sh all detect this alias and route + # their per-file code-signing through it; see + # build-extra's please.sh + installer/release.sh. + script="$(cygpath -au "$BUILD_SOURCESDIRECTORY/.azure-pipelines/esrp/windows/esrpsign.sh")" + git config --global alias.signtool "!sh \"$script\"" + git config --global --get alias.signtool + - task: Bash@3 + displayName: 'Apply Windows build patches' + inputs: + targetType: inline + script: | + set -euo pipefail + apply="$(cygpath -au "$BUILD_SOURCESDIRECTORY/.azure-pipelines/scripts/apply-patches.sh")" + patches="$(cygpath -au "$BUILD_SOURCESDIRECTORY/.azure-pipelines/patches/windows")" + bash "$apply" "$patches/build-extra" /usr/src/build-extra + bash "$apply" "$patches/git-sdk" "/$(mingwprefix)" + - task: Bash@3 + displayName: 'Build mingw-w64-git package' + env: + # Git v2.55 still builds without Rust. + NO_RUST: ForNow + # The mingw-w64-git build is heavy on parallel work + # that the underlying compile (and especially the + # contrib + doc + i18n + perl-script generation + # stages) can soak up far more aggressively than the + # core count would suggest, since most of it is I/O + # against the SDK's pacman cache and the make rules + # have very few real serialisation points. + MAKEFLAGS: -j15 + ${{ if eq(parameters.esrp, true) }}: + ESRP_TOOL: $(ESRP_TOOL) + ESRP_AUTH: $(ESRP_AUTH) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + targetType: inline + script: | + set -euo pipefail + set -x + + # Detach stdin so descendants like the git-extra + # post_install hook (which runs `for s in $(grep + # -l PAT $(find /mingw*/bin/ ...))` and falls back + # to reading stdin when /mingw*/bin/ is absent + # and find produces empty output) cannot block + # the build waiting for input. Bash@3 leaves the + # task's stdin pipe open with no writer; the + # GitHub Actions runner closes it for the same + # reason (see actions/runner ProcessInvoker.cs). + exec /usr/bin/git + chmod +x /usr/bin/git + + USER_NAME='microsoft-git-build' + USER_EMAIL='microsoft-git-build@users.noreply.github.com' + git config --global user.name "$USER_NAME" + git config --global user.email "$USER_EMAIL" + export PACKAGER="$USER_NAME <$USER_EMAIL>" + + # please.sh build-mingw-w64-git derives the package + # version by running `git for-each-ref --points-at=HEAD + # 'refs/tags/v[0-9]*'` against this source tree (and + # then pushing the chosen tag into the freshly cloned + # /usr/src/MINGW-packages/mingw-w64-git/{git,src/git} + # worktrees that makepkg-mingw actually builds from). + # If no v[0-9]* tag is found at HEAD, please.sh falls + # back to `git describe + timestamp` and creates a + # lightweight tag whose name has nothing to do with + # the pipeline-resolved $(git_version). On a real + # release run HEAD is already tagged via the tag-push + # trigger; on a debug run (e.g. the TO-DROP static- + # version override that only fakes git_version in the + # prereqs stage) it is not, so create an annotated + # tag at HEAD ourselves so please.sh picks it up and + # the resulting mingw-w64-git package, the + # cv2pdb-stripped binaries inside it, and the + # downstream Inno Setup installer are all named after + # the resolved git_version. If the tag already exists + # but does not point at HEAD, fail loudly: silently + # skipping creation would let please.sh's for-each-ref + # not find it (since it is not at HEAD) and fall back + # to the describe+timestamp path this whole block is + # supposed to prevent. + if git rev-parse --verify "refs/tags/v$(git_version)" + then + tag_commit="$(git rev-parse "refs/tags/v$(git_version)^{commit}")" + head_commit="$(git rev-parse HEAD)" + test "$tag_commit" = "$head_commit" || { + echo "##vso[task.logissue type=error]Tag v$(git_version) points at $tag_commit, not HEAD ($head_commit)" >&2 + exit 1 + } + else + git tag -a -m "v$(git_version)" "v$(git_version)" HEAD + fi + + # Pre-create the worktree that makepkg-mingw will + # actually compile, instead of letting please.sh's + # `clone --reference $bare https://github.com/git-for-windows/git` + # do it. The PKGBUILD's `build()` -> `make` invokes + # microsoft/git's GIT-VERSION-GEN, which `git + # describe`s in $srcdir/git and rejects any version + # whose `${VN%%.vfs.*}` does not match the hard-coded + # `${DEF_VER%%.vfs.*}` (`v2.53.0`). For real release + # tags the two match, but for any debug tag (in + # particular the v9.99.99.vfs.0.0 the TO-DROP commit + # produces) the build aborts with "Found version + # v9.99.99.vfs.0.0, which is not based on + # v2.53.0.vfs.0.0". GIT-VERSION-GEN reads a `version` + # file at the source-tree root in preference to + # running git describe, and the validation only + # fires on the describe path; so plant that file + # ahead of the build. + # + # please.sh would clone /usr/src/MINGW-packages + # itself if missing; we do the same clone first so + # the package directory exists and please.sh skips + # its own clone (line 838-840 of please.sh). + test -d /usr/src/MINGW-packages || + git clone --depth 1 --single-branch -b main \ + https://github.com/git-for-windows/MINGW-packages \ + /usr/src/MINGW-packages + + # mingw-w64-git/src/git is what makepkg's extract_git + # `git fetch`s and `git checkout --force --no-track + # -B makepkg `s into. Make it a worktree of this + # agent's checkout so the v$VERSION tag we just + # created is already visible there (worktrees share + # refs and objects with the main repo) and we don't + # have to duplicate the source. + # + # Worktrees also share `.git/config` with the main + # repo, so origin would point at the Azure-supplied + # remote URL and extract_git's `git fetch` would go + # online (and abort the build if it failed). Enable + # `extensions.worktreeConfig` and override origin to + # the local checkout via `git config --worktree` so + # the fetch stays on local disk and is effectively a + # no-op. Mirror please.sh's `core.autoCRLF=false` + # under the same per-worktree namespace so the + # checksums calc_checksum_git() computes against + # this tree are reproducible regardless of the main + # repo's autocrlf setting. + mkdir -p /usr/src/MINGW-packages/mingw-w64-git/src + test -d /usr/src/MINGW-packages/mingw-w64-git/src/git || { + git -C "$BUILD_SOURCESDIRECTORY" \ + config extensions.worktreeConfig true + git -C "$BUILD_SOURCESDIRECTORY" worktree add \ + /usr/src/MINGW-packages/mingw-w64-git/src/git HEAD + git -C /usr/src/MINGW-packages/mingw-w64-git/src/git \ + config --worktree remote.origin.url \ + "$BUILD_SOURCESDIRECTORY" + git -C /usr/src/MINGW-packages/mingw-w64-git/src/git \ + config --worktree core.autoCRLF false + } + + # The actual `version` file write that side-steps + # GIT-VERSION-GEN's validation. The file is untracked, + # so makepkg's `git checkout --force --no-track -B + # makepkg ` does not remove it. + echo "$BUILD_VERSION" \ + >/usr/src/MINGW-packages/mingw-w64-git/src/git/version + + sh -x /usr/src/build-extra/please.sh build-mingw-w64-git \ + --only-"$(cpu_arch)" \ + -o artifacts \ + HEAD + + # NOTE: the GitHub workflow additionally GPG-signs + # each tarball and creates a MINGW-packages.bundle + # for downstream Pacman consumers; both are + # intentionally out of scope for the initial port + # and tracked as follow-ups. + - task: Bash@3 + displayName: 'Build installer and portable Git' + env: + # `please.sh make_installers_from_mingw_w64_git` invokes + # build-extra's installer/release.sh, which requires + # MSYSTEM to select the architecture branch. Bash@3 does + # not source /etc/profile, so we export it explicitly. + MSYSTEM: ${{ dim.msystem }} + ${{ if eq(parameters.esrp, true) }}: + ESRP_TOOL: $(ESRP_TOOL) + ESRP_AUTH: $(ESRP_AUTH) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + targetType: inline + script: | + set -euo pipefail + set -x + + # Detach stdin so descendants like the git-extra + # post_install hook (which runs `for s in $(grep + # -l PAT $(find /mingw*/bin/ ...))` and falls back + # to reading stdin when /mingw*/bin/ is absent + # and find produces empty output) cannot block + # the build waiting for input. Bash@3 leaves the + # task's stdin pipe open with no writer; the + # GitHub Actions runner closes it for the same + # reason (see actions/runner ProcessInvoker.cs). + exec /tmp/setx.sh + export BASH_ENV=/tmp/setx.sh + + # please.sh make_installers_from_mingw_w64_git + # --include-pdbs reads PDB archives from + # cached-source-packages/. + mkdir -p "$b/cached-source-packages" + cp artifacts/*-pdb* "$b/cached-source-packages/" + + # The --pkg=... list excludes the optional pieces + # the workflow drops (signatures, archimport, cvs, + # p4, gitweb, doc-man); keep the same filter so + # the resulting .exe size is comparable. + pkg_args=$( + ls artifacts/mingw-w64-$(toolchain)-*.tar.* \ + | sed '/\.sig$/d;/archimport/d;/cvs/d;/p4/d;/gitweb/d;/doc-man/d;s/^/--pkg=/' \ + | tr '\n' ' ' + ) + + for type in installer portable; do + eval sh -x "$b"/please.sh make_installers_from_mingw_w64_git --include-pdbs \ + --version="$(git_version)" \ + -o artifacts --"$type" \ + $pkg_args + + # The installer .exe is signed inline by Inno + # Setup via the `git signtool` alias; the + # portable .exe is a 7z self-extractor that + # bypasses that path, so sign it explicitly. + if test "$type" = portable && \ + test -n "$(git config alias.signtool)" + then + git signtool artifacts/PortableGit-*.exe + fi + done + - task: Bash@3 + displayName: 'Stage installer artifacts for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + + # Compute SHA-256 over the (possibly signed) + # binaries; if ESRP signing ran, this picks up + # the post-sign bytes, which is what we want to + # publish in the release notes. + openssl dgst -sha256 \ + artifacts/Git-*.exe \ + artifacts/PortableGit-*.exe \ + | sed 's/.* //' >artifacts/sha-256.txt + + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + cp artifacts/Git-*.exe \ + artifacts/PortableGit-*.exe \ + artifacts/sha-256.txt \ + "$(Build.ArtifactStagingDirectory)/_final/" + - task: Bash@3 + displayName: 'Extract mingw-w64-git packages for binary analysis' + inputs: + targetType: inline + script: | + set -euo pipefail + + # Stage only the first-party pacman packages produced by + # `please.sh build-mingw-w64-git` for BinSkim, rather + # than the full portable Git installer. This narrows + # the analysis target to binaries this repo's Makefile + # actually builds, and avoids dragging in the third + # party payload (MSYS2/MinGW runtime, Perl, Tcl/Tk, + # GCM, Git LFS, build-extra launcher shims, ...) that + # the installer otherwise bundles. + # + # The three packages extracted are: + # mingw-w64--git--1-any.pkg.tar.xz + # The main git package: git.exe, the dashed + # subcommands, scalar.exe, headless-git.exe, + # git-gvfs-helper.exe, and all other PROGRAMS / + # EXTRA_PROGRAMS the Makefile installs. + # mingw-w64--git-credential-wincred--1-any.pkg.tar.xz + # contrib/credential/wincred/git-credential-wincred.exe + # mingw-w64--git-pdb--1-any.pkg.tar.xz + # cv2pdb-generated .pdb files for the above. These + # are required for several BinSkim checks + # (otherwise we get ERR997.ExceptionLoadingPdb on + # every binary). + # + # The other artifacts from the build (git-archimport, + # git-cvs, git-doc-*, git-for-windows-addons, git-gui, + # git-p4, git-perl, git-send-email, git-subtree, + # git-svn, gitk, gitweb) contain only docs or + # interpreted scripts (Perl/Tcl/Python/sh) and ship + # no native PE binaries built from this repo, so they + # are not staged. + bin="$(Build.ArtifactStagingDirectory)/_bin" + # $(Build.ArtifactStagingDirectory) substitutes a + # Windows-style path with backslashes (e.g. + # D:\a\_work\1\a), producing the mixed-separator + # value D:\a\_work\1\a/_bin. When MSYS2 bash later + # exec()s native Windows utilities like tar.exe, + # its argv path-conversion layer treats such + # arguments as printf-style format strings and + # mangles \a / \1 / etc. into BEL / SOH (0x01), + # so tar's `-C "$bin"` fails with "Cannot open: No + # such file or directory". Normalise to forward + # slashes up front so the path is unambiguous to + # both bash and the MSYS2 runtime. + bin="${bin//\\//}" + mkdir -p "$bin" + + shopt -s nullglob + pkgs=( + artifacts/mingw-w64-*-git-[0-9]*-1-any.pkg.tar.xz + artifacts/mingw-w64-*-git-credential-wincred-[0-9]*-1-any.pkg.tar.xz + artifacts/mingw-w64-*-git-pdb-[0-9]*-1-any.pkg.tar.xz + ) + if test "${#pkgs[@]}" -ne 3 + then + echo "##vso[task.logissue type=error]Expected 3 first-party mingw-w64-git packages in artifacts/, found ${#pkgs[@]}" >&2 + ls -la artifacts/ >&2 + exit 1 + fi + + for pkg in "${pkgs[@]}"; do + name=$(basename "$pkg") + echo "##[group]Extracting $name" + # List the package's PE binaries (and .pdbs) + # before extracting, so the log stays focused on + # what BinSkim will see. `|| true` covers the + # "no match" exit from grep without masking tar + # failures (the following `tar -xf` runs + # independently and will fail loudly under set + # -e if the archive is corrupt). + tar -tf "$pkg" \ + | grep -iE '\.(exe|dll|pdb)$' || true + tar -xf "$pkg" -C "$bin" + echo "##[endgroup]" + done + + # Drop pacman's package-level metadata files; they + # are not binaries and only clutter the staged tree. + rm -f "$bin"/.PKGINFO "$bin"/.MTREE \ + "$bin"/.BUILDINFO "$bin"/.INSTALL + + echo "##[group]All extracted PE binaries (.dll, .exe)" + find "$bin" -type f \( -iname '*.exe' -o -iname '*.dll' \) | sort + echo "##[endgroup]" + # Validate the freshly built installer in-place: silently + # install Git-*.exe and assert that `git --version` reports + # the version we resolved at the prereqs stage. Folded into + # the build job so it runs on the same agent without the + # 1ES job-startup overhead a separate validate job carries. + - powershell: | + $exe = Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)\_final\Git-*.exe" | + Where-Object { $_.Name -notlike 'PortableGit-*' } | + Select-Object -First 1 -ExpandProperty FullName + if (-not $exe) { + Write-Error "No Git-*.exe installer found in _final" + exit 1 + } + Write-Host "Installing $exe" + $p = Start-Process -Wait -PassThru -FilePath "$exe" ` + -ArgumentList "/SILENT","/VERYSILENT","/NORESTART","/SUPPRESSMSGBOXES","/ALLOWDOWNGRADE=1" + if ($p.ExitCode -ne 0) { + Write-Error "Installer exited with code $($p.ExitCode)" + exit $p.ExitCode + } + displayName: 'Install Git' + - powershell: | + $raw = & "$env:ProgramW6432\Git\cmd\git.exe" --version + $actual = ($raw -replace '^git version ', '').Trim() + $expect = ('$(git_version)' -replace '-rc', '.rc').Trim() + Write-Host "Expected: $expect" + Write-Host "Actual: $actual" + if ($actual -ne $expect) { + Write-Error "Version mismatch: expected '$expect', got '$actual'" + exit 1 + } + displayName: 'Validate installed version' + + # + # macOS build jobs + # + - ${{ each dim in parameters.macos_matrix }}: + - job: ${{ dim.id }} + displayName: ${{ dim.jobName }} + pool: + name: ${{ dim.pool }} + image: ${{ dim.image }} + os: ${{ dim.os }} + variables: + tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] + tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] + git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/_final' + artifactName: '${{ dim.id }}' + # macOS build flow: + # + # 1. Configure for an ARM64 build and produce + # Git's own dist tarballs (`make dist dist-doc`). + # 2. Extract the source tarball into payload/, copy + # config.mak in, run `make payload` to compile and + # install into stage/git-arm64-/. + # 3. Mirror stage/ into build-artifacts/ (which is + # what the macos-installer Makefile's `pkg` target + # consumes - see note in the build step). + # 4. ESRP-sign Mach-O files in build-artifacts/. + # 5. `make pkg` -> unsigned .pkg in disk-image/. + # 6. ESRP-sign and ESRP-notarize the .pkg in place. + # 7. `make image` wraps disk-image/ contents in a DMG. + # 8. Stage the .pkg and .dmg under _final/ for upload. + steps: + - checkout: self + - task: Bash@3 + displayName: 'Disable Spotlight indexing' + inputs: + targetType: inline + script: | + # Disable Spotlight indexing to prevent file + # locking issues. + set -euo pipefail + sudo mdutil -i off / || true + - task: Bash@3 + displayName: 'Install build dependencies' + inputs: + targetType: inline + script: | + set -euo pipefail + + # Ensure native Homebrew (arm64) is up-to-date beforehand. + HOMEBREW_NO_AUTO_UPDATE='' brew update + + # Native (arm64) build dependencies. + brew install automake asciidoc xmlto docbook gettext + brew link --force gettext + + # Use the native static libintl.a. It depends on + # the system's /usr/lib/libiconv.dylib rather than + # Homebrew's incompatible _libiconv* symbols. + cp "$(brew --prefix gettext)/lib/libintl.a" libintl.a + - task: Bash@3 + displayName: 'Configure ARM64 build' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + # Git's GIT-VERSION-GEN expects .rc rather than -rc + BUILD_VERSION="$VERSION" + echo "$BUILD_VERSION" >version + + cat >config.mak <>config.mak <>config.mak <>config.mak + - task: Bash@3 + displayName: 'Build payload via macos-installer' + env: + # The macos-installer Makefile derives BUILD_DIR from + # $(GITHUB_WORKSPACE), which is unset in ADO. Point it + # at the worktree root. + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + BUILD_VERSION="$VERSION" + + # The asciidoc/xmlto build steps need the catalogs + # from Homebrew docbook. + export XML_CATALOG_FILES="$(brew --prefix)/etc/xml/catalog" + + # `git commit` (in dist-doc) forks a detached + # `git maintenance run --auto` that keeps writing + # into .git/ after the commit returns, which then + # races with dist-doc's `rm -fr .doc-tmp-dir` and + # produces "Directory not empty". Disable + # auto-maintenance for every git invocation in + # this build. + export GIT_CONFIG_PARAMETERS="'maintenance.auto=false'" + + rustup target add aarch64-apple-darwin + export RUST_TARGETS=aarch64-apple-darwin + + make -j"$(sysctl -n hw.physicalcpu)" GIT-VERSION-FILE dist dist-doc + + # Recover the source-tree commit OID from the dist + # tarball; the macos-installer Makefile bakes it + # into 'git version --build-options'. + # `git get-tar-commit-id` reads only the leading + # pax header and then closes its stdin, which + # makes `gunzip -c` exit 141 (SIGPIPE) under the + # outer `set -o pipefail`. Disable pipefail for + # the duration of this one pipeline. + GIT_BUILT_FROM_COMMIT="$( + set +o pipefail + gunzip -c "git-$BUILD_VERSION.tar.gz" | + git get-tar-commit-id + )" + export GIT_BUILT_FROM_COMMIT + export VERSION + + mkdir payload manpages + tar -xf "git-$BUILD_VERSION.tar.gz" -C payload + tar -xf "git-manpages-$BUILD_VERSION.tar.gz" -C manpages + + # The actual compile happens inside the extracted + # tree, against a copy of the config.mak we wrote + # at the worktree root in the previous step. + cp config.mak "payload/git-$BUILD_VERSION/config.mak" + + make -C .github/macos-installer V=1 \ + ARCH_UNIV=arm64 ARCH_FLAGS="-arch arm64" payload + + # NOTE: the macos-installer Makefile produces + # the install tree at stage/git-arm64-/ + # but its `pkg` target packages from + # build-artifacts/. Copy the tree across so the + # pipeline (signing, pkg) finds it where the + # Makefile expects. + # + # FUTURE: this duplication exists only because + # .github/macos-installer/Makefile hardcodes + # both DESTDIR (=stage/...) and ARTIFACTDIR + # (=build-artifacts). Overriding ARTIFACTDIR on + # the `make pkg` line below to point at stage/ + # would let us drop the cp entirely. Worth + # cleaning up alongside moving the macOS + # installer Makefiles out of .github/ (they are + # build infrastructure, not GitHub-specific). + mkdir -p .github/macos-installer/build-artifacts + cp -R "stage/git-arm64-$BUILD_VERSION/." \ + .github/macos-installer/build-artifacts/ + # ESRP-sign the ARM64 Mach-O binaries inside the + # payload tree. The existing esrp/sign.yml template's + # CopyFiles@2 step uses minimatch globs and the only + # reliable way to detect Mach-O is by file content + # (file --mime), so we pre-filter into a staging dir, + # let the template zip/sign/extract that staging dir, + # then copy the signed binaries back over the payload. + - ${{ if eq(parameters.esrp, true) }}: + # ESRP ADO tasks require .NET, which the macOS pool + # image does not provide by default. + - task: UseDotNet@2 + displayName: 'Install .NET for ESRP' + inputs: + packageType: sdk + version: '8.x' + - task: Bash@3 + displayName: 'Stage Mach-O binaries for signing' + inputs: + targetType: inline + script: | + set -euo pipefail + + # Sign the install tree (build-artifacts/) - + # this is what `make pkg` packages. Signing + # the source tree under payload/ would have + # no effect on the resulting .pkg. + install_tree=".github/macos-installer/build-artifacts/usr/local/git" + stage_dir="$(Build.ArtifactStagingDirectory)/macos-tosign/binaries" + + rm -rf "$stage_dir" + mkdir -p "$stage_dir" + + pushd "$install_tree" + find . -type f -exec file --mime {} + \ + | sed -n '/mach/s/: .*//p' \ + | while IFS= read -r f; do + rel="${f#./}" + tgt="$stage_dir/$rel" + mkdir -p "$(dirname "$tgt")" + cp -- "$f" "$tgt" + done + popd + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-sign Mach-O binaries' + folderPath: '$(Build.ArtifactStagingDirectory)/macos-tosign/binaries' + pattern: '**/*' + useArchive: true # Required for macOS signing + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppDeveloperSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "Hardening": "Enable" + } + } + ] + - task: Bash@3 + displayName: 'Copy signed binaries back to install tree' + inputs: + targetType: inline + script: | + set -euo pipefail + + cp -R "$(Build.ArtifactStagingDirectory)/macos-tosign/binaries"/* \ + .github/macos-installer/build-artifacts/usr/local/git/ + - task: Bash@3 + displayName: 'Build unsigned installer pkg' + env: + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + export VERSION + + # Leave APPLE_INSTALLER_IDENTITY undefined so the + # Makefile's `pkg` target produces an unsigned .pkg + # (the `ifdef APPLE_INSTALLER_IDENTITY` branch in + # pkg_cmd is skipped). ESRP signs it in the next + # step. + make -C .github/macos-installer V=1 ARCH_UNIV=arm64 pkg + - ${{ if eq(parameters.esrp, true) }}: + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-sign installer pkg' + folderPath: '.github/macos-installer/disk-image' + pattern: '*.pkg' + useArchive: true # Required for macOS signing + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppDeveloperSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "Hardening": "Enable" + } + } + ] + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-notarize installer pkg' + folderPath: '.github/macos-installer/disk-image' + pattern: '*.pkg' + useArchive: true # Required for macOS notarization + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppNotarize", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "BundleId": "com.git.pkg" + } + } + ] + - task: Bash@3 + displayName: 'Build DMG' + env: + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + export VERSION + + # Builds git--arm64.dmg from disk-image/, + # which contains the signed and notarized .pkg. + make -C .github/macos-installer V=1 ARCH_UNIV=arm64 image + - task: Bash@3 + displayName: 'Stage installer artifacts for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + ls -la .github/macos-installer/ \ + .github/macos-installer/disk-image/ || true + # The .pkg lands either directly under disk-image/ + # or, after ESRP MacAppNotarize re-packs it, inside + # disk-image/.zip.unzipped/. Find it. + pkg=$(find .github/macos-installer/disk-image \ + -name 'git-*-arm64.pkg' -type f \ + | head -1) + mv .github/macos-installer/git-*-arm64.dmg \ + "$pkg" \ + "$(Build.ArtifactStagingDirectory)/_final/" + # Validate the freshly built pkg in-place: install it, + # assert `git --version`, and confirm the binary contains + # only the ARM64 architecture. Folded into the build job + # so it runs on the same agent without the 1ES job-startup + # overhead a separate validate job carries. + - bash: | + set -e + if [ "$(uname -m)" = arm64 ] && command -v brew >/dev/null; then + brew uninstall git || true + fi + pkg=$(find "$(Build.ArtifactStagingDirectory)/_final" \ + -name 'git-*-arm64.pkg' -type f | head -1) + if [ -z "$pkg" ]; then + echo "No git-*-arm64.pkg found in _final" >&2 + exit 1 + fi + echo "Installing $pkg" + sudo installer -pkg "$pkg" -target / + displayName: 'Install Git' + - bash: | + set -e + actual=$(git --version | sed 's/^git version //') + expect="$(git_version)" + echo "Expected: $expect" + echo "Actual: $actual" + test "$actual" = "$expect" + displayName: 'Validate installed version' + - bash: | + set -ex + git version --build-options >actual + cat actual + grep '^cpu: arm64$' actual + git_path="$(command -v git)" + test "$(lipo -archs "$git_path")" = arm64 + displayName: 'Validate ARM64 binary CPU architecture' + + # + # Linux build jobs + # + - ${{ each dim in parameters.linux_matrix }}: + - job: ${{ dim.id }} + displayName: ${{ dim.jobName }} + pool: + name: ${{ dim.pool }} + image: ${{ dim.image }} + os: ${{ dim.os }} + hostArchitecture: ${{ dim.poolArch }} + variables: + tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] + tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] + git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] + cc_arch: ${{ dim.cc_arch }} + deb_arch: ${{ dim.deb_arch }} + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/_final' + artifactName: '${{ dim.id }}' + steps: + - checkout: self + - task: Bash@3 + displayName: 'Log build environment' + inputs: + targetType: inline + script: | + lsb_release -a || true + uname -a + id + - task: Bash@3 + displayName: 'Install build dependencies' + inputs: + targetType: inline + script: | + set -euo pipefail + # The 1ES Ubuntu agents come up with `unattended- + # upgrades` running, which holds the dpkg + # frontend lock for the first few minutes after + # boot. `apt-get` releases earlier than 2.1 + # would have failed immediately with + # E: Could not get lock /var/lib/dpkg/lock-frontend + # Pass `DPkg::Lock::Timeout=600` so apt waits up + # to 10 minutes for the lock instead. + sudo apt-get -o DPkg::Lock::Timeout=600 update -q + sudo apt-get -o DPkg::Lock::Timeout=600 install -y -q --no-install-recommends \ + build-essential \ + tcl tk gettext asciidoc xmlto \ + libcurl4-gnutls-dev libpcre2-dev zlib1g-dev libexpat-dev \ + curl ca-certificates cargo + - task: Bash@3 + displayName: 'Build microsoft-git Debian package' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + # Git's GIT-VERSION-GEN expects .rc rather than -rc + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + echo "$BUILD_VERSION" >version + make GIT-VERSION-FILE + + PKGNAME="microsoft-git_${VERSION}_$(deb_arch)" + PKGDIR="$(Build.ArtifactStagingDirectory)/pkgroot" + rm -rf "$PKGDIR" + mkdir -p "$PKGDIR/DEBIAN" + + DESTDIR="$PKGDIR" make -j"$(nproc)" V=1 DEVELOPER=1 \ + USE_LIBPCRE=1 \ + USE_CURL_FOR_IMAP_SEND=1 NO_OPENSSL=1 \ + NO_CROSS_DIRECTORY_HARDLINKS=1 \ + ASCIIDOC8=1 ASCIIDOC_NO_ROFF=1 \ + ASCIIDOC='TZ=UTC asciidoc' \ + prefix=/usr/local \ + gitexecdir=/usr/local/lib/git-core \ + libexecdir=/usr/local/lib/git-core \ + htmldir=/usr/local/share/doc/git/html \ + install install-doc install-html + + # Based on https://packages.ubuntu.com/xenial/vcs/git + cat >"$PKGDIR/DEBIAN/control" < + Description: Git client built from the https://github.com/microsoft/git repository, + specialized in supporting monorepo scenarios. Includes the Scalar CLI. + CTRL + + mkdir -p "$(Build.ArtifactStagingDirectory)/app" + dpkg-deb -Zxz --build "$PKGDIR" \ + "$(Build.ArtifactStagingDirectory)/app/$PKGNAME.deb" + - ${{ if eq(parameters.esrp, true) }}: + # ESRP ADO tasks require .NET, so we install it here since the + # Linux images do not have it by default. + - task: UseDotNet@2 + displayName: 'Install .NET for ESRP' + inputs: + packageType: sdk + version: '8.x' + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'Sign Debian package' + folderPath: '$(Build.ArtifactStagingDirectory)/app' + pattern: '**/*.deb' + inlineOperation: | + [ + { + "KeyCode": "CP-500207-Pgp", + "OperationCode": "LinuxSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": {} + } + ] + - task: Bash@3 + displayName: 'Stage Debian package for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + mv "$(Build.ArtifactStagingDirectory)/app/microsoft-git_$(git_version)_$(deb_arch).deb" \ + "$(Build.ArtifactStagingDirectory)/_final/" + # Validate the freshly built .deb in-place: install it + # and assert `git --version`. Folded into the build job + # so it runs on the same agent without the 1ES job- + # startup overhead a separate validate job carries. + - bash: | + set -e + deb=$(find "$(Build.ArtifactStagingDirectory)/_final" \ + -name 'microsoft-git_*.deb' -type f | head -1) + if [ -z "$deb" ]; then + echo "No microsoft-git_*.deb found in _final" >&2 + exit 1 + fi + echo "Installing $deb" + # Wait up to 10 minutes for unattended-upgrades to + # release the dpkg lock; see comment on this job's + # 'Install build dependencies' step. + sudo apt-get -o DPkg::Lock::Timeout=600 update + sudo apt-get -o DPkg::Lock::Timeout=600 install -y "$deb" + displayName: 'Install Git' + - bash: | + set -e + actual=$(git --version | sed 's/^git version //') + expect="$(git_version)" + echo "Expected: $expect" + echo "Actual: $actual" + test "$actual" = "$expect" + displayName: 'Validate installed version' + + - stage: release + displayName: 'Release' + dependsOn: [prereqs, build] + jobs: + # + # GitHub release publishing + # + - job: github + displayName: 'Publish GitHub release' + condition: and(succeeded(), eq('${{ parameters.github }}', true), or(eq('${{ parameters.versionOverride }}', ''), eq('${{ parameters.versionOverride }}', '-'))) + pool: + name: GitClientPME-1ESHostedPool-intel-pc + image: ubuntu-x86_64-ado1es + os: linux + variables: + tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] + tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] + git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] + templateContext: + type: releaseJob + isProduction: true + inputs: + - ${{ each dim in parameters.windows_matrix }}: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + - ${{ each dim in parameters.macos_matrix }}: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + - ${{ each dim in parameters.linux_matrix }}: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + steps: + - task: GitHubRelease@1 + displayName: 'Create Draft GitHub Release' + inputs: + gitHubConnection: $(githubConnectionName) + repositoryName: microsoft/git + tag: '$(tag_name)' + tagSource: userSpecifiedTag + target: '$(tag_sha)' + title: '$(tag_name)' + isDraft: true + addChangeLog: true + assets: | + $(Pipeline.Workspace)/assets/windows_x64/*.exe + $(Pipeline.Workspace)/assets/windows_x64/*.zip + $(Pipeline.Workspace)/assets/windows_arm64/*.exe + $(Pipeline.Workspace)/assets/windows_arm64/*.zip + $(Pipeline.Workspace)/assets/macos_arm64/*.pkg + $(Pipeline.Workspace)/assets/macos_arm64/*.dmg + $(Pipeline.Workspace)/assets/macos_arm64/*.tar.gz + $(Pipeline.Workspace)/assets/linux_x64/*.deb + $(Pipeline.Workspace)/assets/linux_x64/*.tar.gz + $(Pipeline.Workspace)/assets/linux_arm64/*.deb + $(Pipeline.Workspace)/assets/linux_arm64/*.tar.gz diff --git a/.azure-pipelines/scripts/apply-patches.sh b/.azure-pipelines/scripts/apply-patches.sh new file mode 100755 index 00000000000000..f1325a7caa0064 --- /dev/null +++ b/.azure-pipelines/scripts/apply-patches.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Apply all numbered patches from a directory to a target tree. +# +# Patches are applied in lexicographic order, so name them with +# zero-padded numeric prefixes (e.g. 0000-foo.patch, 0001-bar.patch). +# +# Arguments: +# $1 patches_dir Directory containing *.patch files +# $2 target_dir Directory to apply patches in (need not be a +# git repository; git apply works on any tree) + +set -euo pipefail + +if test $# -ne 2 +then + echo "Usage: $0 " >&2 + exit 1 +fi + +patches_dir="$1" +target_dir="$2" + +if test ! -d "$patches_dir" +then + echo "Patches directory not found: $patches_dir" >&2 + exit 1 +fi + +if test ! -d "$target_dir" +then + echo "Target directory not found: $target_dir" >&2 + exit 1 +fi + +shopt -s nullglob +patches=("$patches_dir"/*.patch) +if test ${#patches[@]} -eq 0 +then + echo "No patches found in $patches_dir" + exit 0 +fi + +cd "$target_dir" +for patch in "${patches[@]}" +do + echo "Applying $(basename "$patch")..." + # Use patch(1) rather than `git apply` because the latter is + # strict about context whitespace; CRLF/LF mismatches between + # patch context (as authored) and the working tree (which may + # be CRLF on Windows checkouts) trip it up. patch is more + # forgiving by default. + # + # This matches the convention used by msys2/MINGW-packages + # PKGBUILDs and git-for-windows/build-extra's get-sources.sh. + command patch -p1 -i "$patch" +done diff --git a/.azure-pipelines/scripts/resolve-version.sh b/.azure-pipelines/scripts/resolve-version.sh new file mode 100755 index 00000000000000..6169ae767bae1f --- /dev/null +++ b/.azure-pipelines/scripts/resolve-version.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Resolve version and tag information from the current HEAD commit. +# Validates that HEAD is an annotated version tag matching GIT-VERSION-GEN. +# +# Sets the following ADO output variables (via ##vso): +# git_version - Version string without "v" prefix (e.g., 2.53.0.vfs.0.0) +# tag_name - Full tag name (e.g., v2.53.0.vfs.0.0) +# tag_sha - Commit SHA of HEAD +# +# Also updates the build number to include the tag name. +# +set -euo pipefail + +echo "HEAD: $(git rev-parse HEAD)" + +# Determine the tag pointing at HEAD +tag_name=$(git describe --exact-match --match "v[0-9]*vfs*" HEAD 2>/dev/null) || { + echo "##vso[task.logissue type=error]HEAD is not tagged with a version tag" + exit 1 +} + +# Verify the tag is annotated (not lightweight) +tag_type=$(git cat-file -t "refs/tags/$tag_name") +if [ "$tag_type" != "tag" ]; then + echo "##vso[task.logissue type=error]Tag $tag_name is not annotated (type: $tag_type)" + exit 1 +fi + +tag_sha=$(git rev-parse HEAD) +git_version="${tag_name#v}" + +# Verify the version matches GIT-VERSION-GEN +make GIT-VERSION-FILE +expected_version="${git_version//-rc/.rc}" +actual_version=$(sed -n 's/^GIT_VERSION *= *//p' < GIT-VERSION-FILE) +if [ "$expected_version" != "$actual_version" ]; then + echo "##vso[task.logissue type=error]GIT-VERSION-FILE ($actual_version) does not match tag $tag_name ($expected_version)" + exit 1 +fi + +echo "Git version: $git_version" +echo "Tag name: $tag_name" +echo "Tag SHA: $tag_sha" +echo "##vso[task.setvariable variable=git_version;isOutput=true;isReadOnly=true]$git_version" +echo "##vso[task.setvariable variable=tag_name;isOutput=true;isReadOnly=true]$tag_name" +echo "##vso[task.setvariable variable=tag_sha;isOutput=true;isReadOnly=true]$tag_sha" +echo "##vso[build.updatebuildnumber]${tag_name} (${BUILD_BUILDNUMBER:-unknown})" diff --git a/.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 b/.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 new file mode 100644 index 00000000000000..a035f43897e4b8 --- /dev/null +++ b/.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 @@ -0,0 +1,99 @@ +# Set up cv2pdb-strip support on Windows x64 agents. +# +# build-extra's please.sh runs cv2pdb-strip during the strip phase of +# build-mingw-w64-git. cv2pdb-strip loads mspdb140.dll via PATH +# lookup, and the DLL is part of the MSVC C++ toolchain +# (Microsoft.VisualStudio.Component.VC.Tools.x86.x64) which is not +# present on the 1ES image by default. +# +# Install VS 2022 Build Tools with that single component (the +# smallest selection that ships the DLL), locate mspdb140.dll via +# vswhere with a filesystem fallback, and prepend its directory to +# PATH for subsequent tasks via the `##vso[task.prependpath]` logging +# command. +# +# This script is intended to be invoked by a PowerShell@2 task with +# `filePath:`. It takes no arguments and writes diagnostics to stdout +# so install failures can be diagnosed from the task log. + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +$bootstrapper = "$env:TEMP\vs_BuildTools.exe" +Write-Host "Downloading VS 2022 Build Tools bootstrapper..." +Invoke-WebRequest -Uri 'https://aka.ms/vs/17/release/vs_BuildTools.exe' ` + -OutFile $bootstrapper + +$vsArgs = @( + '--quiet', '--wait', '--norestart', '--nocache', + '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' +) +Write-Host "Installing VS Build Tools (args: $($vsArgs -join ' '))..." +$start = Get-Date +$p = Start-Process -FilePath $bootstrapper -ArgumentList $vsArgs -Wait -PassThru +$elapsed = (Get-Date) - $start +Write-Host ("Installer exited with code {0} after {1:N0}s" -f ` + $p.ExitCode, $elapsed.TotalSeconds) + +Write-Host "" +Write-Host "===== Installer logs in `$env:TEMP =====" +$logs = Get-ChildItem $env:TEMP -Filter 'dd_*.log' -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending +if ($logs) { + foreach ($log in $logs | Select-Object -First 5) { + Write-Host "----- $($log.FullName) (last 50 lines) -----" + Get-Content $log.FullName -Tail 50 -ErrorAction SilentlyContinue + } +} else { + Write-Host "(no dd_*.log files found in `$env:TEMP)" +} + +Write-Host "" +Write-Host "===== vswhere -all -prerelease (every install) =====" +$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path $vswhere)) { + Write-Host "vswhere not found at $vswhere" +} else { + & $vswhere -all -prerelease -format json | + Out-String | Write-Host +} + +Write-Host "" +Write-Host "===== Filesystem search for mspdb*.dll =====" +$roots = @( + "${env:ProgramFiles(x86)}\Microsoft Visual Studio", + "${env:ProgramFiles}\Microsoft Visual Studio" +) | Where-Object { Test-Path $_ } +$hits = foreach ($r in $roots) { + Get-ChildItem -Path $r -Filter 'mspdb*.dll' -Recurse -File ` + -ErrorAction SilentlyContinue +} +if ($hits) { + $hits | ForEach-Object { Write-Host $_.FullName } +} else { + Write-Host "(no mspdb*.dll under any VS install root)" +} + +# 3010 = reboot required, treated as success. +if ($p.ExitCode -notin 0,3010) { + throw "VS Build Tools installer exited with code $($p.ExitCode)" +} + +Write-Host "" +Write-Host "===== Locate mspdb140.dll via vswhere -find =====" +$mspdb = & $vswhere -latest ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -find 'VC\Tools\MSVC\**\bin\Hostx64\x64\mspdb140.dll' | + Select-Object -First 1 +if (-not $mspdb) { + # Fall back to filesystem hits we already have. + $mspdb = $hits | + Where-Object { $_.Name -ieq 'mspdb140.dll' } | + Select-Object -First 1 -ExpandProperty FullName +} +if (-not $mspdb) { + throw "mspdb140.dll not found after install (see logs above)" +} +$dir = Split-Path -Parent $mspdb +Write-Host "Found mspdb140.dll at $mspdb" +Write-Host "##vso[task.prependpath]$dir" diff --git a/.azure-pipelines/scripts/windows/setup-git-bash.cmd b/.azure-pipelines/scripts/windows/setup-git-bash.cmd new file mode 100644 index 00000000000000..b3ef5518cfc85d --- /dev/null +++ b/.azure-pipelines/scripts/windows/setup-git-bash.cmd @@ -0,0 +1,13 @@ +@echo off +setlocal enabledelayedexpansion +set "agentgit=%AGENT_HOMEDIRECTORY%\externals\git" +set "gitcopy=%AGENT_TEMPDIRECTORY%\git" +echo Copying !agentgit! to !gitcopy!... +xcopy /E /I /Q "!agentgit!" "!gitcopy!" +if not exist "!gitcopy!\usr\bin\sh.exe" ( + echo ##vso[task.logissue type=error]Could not find sh.exe at !gitcopy!\usr\bin\sh.exe + exit /b 1 +) +echo Copying !gitcopy!\usr\bin\sh.exe to !gitcopy!\usr\bin\bash.exe... +copy /Y "!gitcopy!\usr\bin\sh.exe" "!gitcopy!\usr\bin\bash.exe" +echo ##vso[task.prependpath]!gitcopy!\usr\bin diff --git a/.azure-pipelines/scripts/windows/setup-git-sdk.sh b/.azure-pipelines/scripts/windows/setup-git-sdk.sh new file mode 100755 index 00000000000000..0634bd61fd7e38 --- /dev/null +++ b/.azure-pipelines/scripts/windows/setup-git-sdk.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Materialise the build-installers flavour of the Git for Windows SDK. +# +# Performs a partial + bare clone of the given Git SDK repository, +# then runs build-extra's please.sh to sparse-checkout just the +# build-installers subset into the requested SDK output directory. +# +# Environment: +# BOOTSTRAP_DIR (optional) - directory for transient bootstrap clones +# (the bare git-sdk fetch and build-extra +# checkout used to drive please.sh). +# Falls back to TEMP, then TMP, then errors +# if none are set. +# +# Arguments: +# $1 sdk_repo e.g. git-for-windows/git-sdk-64 +# $2 mingwprefix e.g. ucrt64 or clangarm64 +# $3 sdk_output_dir Windows or MSYS path where the SDK will be installed +# +# See: +# https://github.com/git-for-windows/git-sdk-64/blob/main/.github/workflows/ci-artifacts.yml +# https://github.com/git-for-windows/build-extra/blob/main/please.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/utils.sh" + +if test $# -ne 3 +then + echo "Usage: $0 " >&2 + exit 1 +fi + +sdk_repo="$1" +mingwprefix="$2" +sdk_output="$3" + +bootstrap_dir="${BOOTSTRAP_DIR:-${TEMP:-${TMP:-}}}" +if test -z "$bootstrap_dir" +then + echo "BOOTSTRAP_DIR (or TEMP/TMP) must be set" >&2 + exit 1 +fi + +bootstrap="$(to_unix_path "$bootstrap_dir")" +sdk="$(to_unix_path "$sdk_output")" + +sdk_bare="$bootstrap/sdk-bare.git" +bootstrap_be="$bootstrap/build-extra-bootstrap" + +git init --bare "$sdk_bare" +git --git-dir="$sdk_bare" remote add origin "https://github.com/$sdk_repo" +git --git-dir="$sdk_bare" config remote.origin.promisor true +git --git-dir="$sdk_bare" config remote.origin.partialCloneFilter blob:none +git --git-dir="$sdk_bare" fetch --depth=1 origin HEAD +git --git-dir="$sdk_bare" update-ref --no-deref HEAD FETCH_HEAD + +# please.sh is the bootstrap; build-extra gets cloned again into the SDK +# in a separate task so `please.sh build-mingw-w64-git` can find it at +# /usr/src/build-extra under the SDK's bash. +git clone --depth=1 --single-branch -b main \ + https://github.com/git-for-windows/build-extra \ + "$bootstrap_be" + +# Architecture is auto-detected from the bare clone's HEAD tree +# (clangarm64/ vs usr/x86_64-pc-msys/). +bash "$bootstrap_be/please.sh" create-sdk-artifact \ + --sdk="$sdk_bare" --out="$sdk" build-installers + +# Expose the SDK's bash and the matching MinGW toolchain to subsequent +# tasks. +echo "##vso[task.prependpath]$(to_windows_path "$sdk/usr/bin")" +echo "##vso[task.prependpath]$(to_windows_path "$sdk/$mingwprefix/bin")" diff --git a/.azure-pipelines/scripts/windows/utils.sh b/.azure-pipelines/scripts/windows/utils.sh new file mode 100755 index 00000000000000..f94c380a2b49af --- /dev/null +++ b/.azure-pipelines/scripts/windows/utils.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Utilities for bash scripts running on Windows. +# +# Source this file from another bash script: +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# . "$SCRIPT_DIR/utils.sh" +# +# Functions: +# to_windows_path - output a Windows-form (D:\foo) path. +# to_unix_path - output an MSYS-form (/d/foo) path. + +# Convert a path to Windows form for tools that demand backslashes +# (e.g. ESRPClient.exe, ##vso[task.prependpath]). +# Useful when a script may run before the full Git for Windows SDK +# (which provides cygpath) is available. Falls back to pure-shell +# parsing when cygpath is not on PATH. +to_windows_path () { + local drive rest root + if command -v cygpath >/dev/null 2>&1; then + cygpath -w "$1" + return + fi + case "$1" in + /[A-Za-z]/*) + # /d/path -> D:\path + drive=$(echo "$1" | cut -c2 | tr 'a-z' 'A-Z') + rest=$(echo "$1" | cut -c3-) + echo "${drive}:${rest}" | sed 's|/|\\|g' + ;; + /*) + # Absolute path under MSYS root + root=$(cd / && pwd -W) + echo "${root}${1}" | sed 's|/|\\|g' + ;; + *) + # Relative or already-Windows: just flip slashes + echo "$1" | sed 's|/|\\|g' + ;; + esac +} + +# Convert a path to MSYS form for bash-friendly handling. Inverse of +# to_windows_path. +# Useful when a script may run before the full Git for Windows SDK +# (which provides cygpath) is available. Falls back to pure-shell +# parsing when cygpath is not on PATH. +to_unix_path () { + local p drive rest + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$1" + return + fi + # Normalize separators to forward slashes first. + p="${1//\\//}" + case "$p" in + [A-Za-z]:/*) + # D:/path -> /d/path + drive=$(echo "$p" | cut -c1 | tr 'A-Z' 'a-z') + rest=$(echo "$p" | cut -c3-) + echo "/${drive}${rest}" + ;; + *) + echo "$p" + ;; + esac +} diff --git a/.azure-pipelines/sdl/windows_arm64/.gdnsuppress b/.azure-pipelines/sdl/windows_arm64/.gdnsuppress new file mode 100644 index 00000000000000..6d8f0dc3e225fa --- /dev/null +++ b/.azure-pipelines/sdl/windows_arm64/.gdnsuppress @@ -0,0 +1,720 @@ +{ + "hydrated": true, + "properties": { + "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/suppressions" + }, + "version": "1.0.0", + "suppressionSets": { + "default": { + "name": "default", + "createdDate": "2026-05-27 10:14:26Z", + "lastUpdatedDate": "2026-05-27 10:14:26Z" + } + }, + "results": { + "700115aaeb52ef14c3ecbe6969846d61952c6d886621015dffde4e3bdb61da19": { + "signature": "700115aaeb52ef14c3ecbe6969846d61952c6d886621015dffde4e3bdb61da19", + "alternativeSignatures": [ + "2e72db9df4196700b91316238124eb512f9d037af53ffd4c9e988d775f4612ed", + "12d9f3c3e169e2b1bf64e764f419192c12f43c401e1d7e4676e230ce9a546875", + "98631e820190d0dce5ee357ff9de64ed1273253648398676b9514615c921b9e6" + ], + "target": "_bin/clangarm64/bin/git-receive-pack.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "b7e5647d97442b6f2c23b33e8aafdf22c0d6aed098df1ef15e571f2d8541f672": { + "signature": "b7e5647d97442b6f2c23b33e8aafdf22c0d6aed098df1ef15e571f2d8541f672", + "alternativeSignatures": [ + "a525b473688cfa00a627005d40c76484e6744ac8f69e7f127d94f6f490768a66", + "beb22705dca5cfc08bc99ace21633b10b4f501734d2fa64a7cc94c05059f0a8b", + "4107569dac44655d3b70ef97a70f38f694693225930b98a3d4b4c9db6bd018a4" + ], + "target": "_bin/clangarm64/bin/git-receive-pack.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "31a757644db7f89cb414cc657f0344879b7bd6d65e06a844d1b3642d49cd29c8": { + "signature": "31a757644db7f89cb414cc657f0344879b7bd6d65e06a844d1b3642d49cd29c8", + "alternativeSignatures": [ + "2c5152ae42f2f059bafdb22e9291914dea12d431d0a572e50253f4feefc08a6c", + "1d6b5dbbfa21ce2fe38254752a6792fd453debe35c0d2471cfa1a0c2de7bc7fe", + "a196e7d2cc2b2556bec6cabcd4bed7c5752f9d1b1e5581bb63867c90333aeedc" + ], + "target": "_bin/clangarm64/bin/git-shell.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "faee4a620858f2b998bb548c3c01188b725e40d0843ce25c0913bb436ec9d99e": { + "signature": "faee4a620858f2b998bb548c3c01188b725e40d0843ce25c0913bb436ec9d99e", + "alternativeSignatures": [ + "294b1ca1a4c3d990940b286f075ff5329070d6d6a7e49e9d498d1d6c8c730b3c", + "88aec4219484bf7a78fe2b9e58801fd15080ede437390b917208c0a0845b313d", + "66a40f17c2b6737636e5a9cc7b10311274e576d26bb83e4880172fcc07ab26cd" + ], + "target": "_bin/clangarm64/bin/git-shell.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "3333d624de5ce21b7c8dc05712537e1e89d244fb52df633e2387b1fed7bb96e0": { + "signature": "3333d624de5ce21b7c8dc05712537e1e89d244fb52df633e2387b1fed7bb96e0", + "alternativeSignatures": [ + "8d3e28c153ee6977b82a460864bd58aabba9f53c325aa2bdde8a1a94abf3565f", + "d45931317dad3d8ff6f9e93daf30f0cf423de5615e0d9601197512c2a9becb1c", + "8f7a6d152ea717e80f9167b5e36d73238b40dc5d7645bdd815d495a2976df982" + ], + "target": "_bin/clangarm64/bin/git-upload-archive.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "c357826a839c5403175734a100b79efea4ba3c2b59d1a60c2893c0a332da0220": { + "signature": "c357826a839c5403175734a100b79efea4ba3c2b59d1a60c2893c0a332da0220", + "alternativeSignatures": [ + "5d95359b2b45e2a5628494b832fe4cc61c07641265fe4224febabf97d51274a8", + "5eba68419d9400267749f597dbf4fa0dc2c13e6d980e43d522a9ccf7da047984", + "e6756716f7c84f9db8ce92d6099eabf631c717fb5e18d84f4b3fce80b3bd9029" + ], + "target": "_bin/clangarm64/bin/git-upload-archive.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "26d93852f06da066529b0e93f720f86d2354de41340f75d503fcd7e26d437a12": { + "signature": "26d93852f06da066529b0e93f720f86d2354de41340f75d503fcd7e26d437a12", + "alternativeSignatures": [ + "8d3113f61cbb9b7706990b5569ed13ec9352ab41dd9f14005306bed0390243b6", + "38ba10bc6b3dfa697c54ff2d4a9055e3eacbdc69a1c255b206e348590482ee45", + "ae349628ed4c5bca9af3462a61bef103ab88aa4586bf46c02206a4b9369d0bae" + ], + "target": "_bin/clangarm64/bin/git-upload-pack.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "ed6be62f4679dfa3b8af8fc358db90c4cba7eebe2c1e61a30154a76535bd20d8": { + "signature": "ed6be62f4679dfa3b8af8fc358db90c4cba7eebe2c1e61a30154a76535bd20d8", + "alternativeSignatures": [ + "4e911ea879e061ad277594770728f912a3206448cf6c77dd1604d6c6fc3f8d5f", + "75abc0a14b3857868f296666c8f6cf8779617be0a0d3d8db3ab3f7928af60675", + "50d16cecea5fb05dd3e589591e7a128f8fa6fa34b04394f8e14a0a607f54d7ae" + ], + "target": "_bin/clangarm64/bin/git-upload-pack.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "7a48b83559cff8b21323116d171cbad033f44d9acaa1d54a78e0f242ff247107": { + "signature": "7a48b83559cff8b21323116d171cbad033f44d9acaa1d54a78e0f242ff247107", + "alternativeSignatures": [ + "474c364b50285737312ad3a443fd17550fc8352c4b04ebbe2d5dc5e0e54cff33", + "3cbdb0cecb6f216f68513823bc7accc4706b262936025ca267b77ad5f929833e", + "923de95322f1a2c95315440cf7ce01be268008fed46ab1e2f35ca6d6bbd3aaff" + ], + "target": "_bin/clangarm64/bin/git.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "29ffaec051c0041cfe5a5aa296ef59c5ae06af8439e484fd38c70af1625d1cd4": { + "signature": "29ffaec051c0041cfe5a5aa296ef59c5ae06af8439e484fd38c70af1625d1cd4", + "alternativeSignatures": [ + "efe8c09007ee71308ed9ecc3cb1fc64524d17e10d66f5e479e7b96b91f165e9e", + "8c305bc7dcf6fb85876fee6d0d65eb0f778550eb4a26748acaa5861c2b5bed0f", + "5cd942e56bcdad24defcd3f73cce8175bc35a3bf8f3330293f13868c0e9c6527" + ], + "target": "_bin/clangarm64/bin/git.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "929aadbdd62c111f4f14f6e9454c2c95c896aee5c683fa7f23a3471e06fbeb11": { + "signature": "929aadbdd62c111f4f14f6e9454c2c95c896aee5c683fa7f23a3471e06fbeb11", + "alternativeSignatures": [ + "319997640211e8d78718de761aedd9bf1a0216468b31a00356185159c87f8951", + "907bbcdbbcbbd2efd732eb076365b6c05e8f3bd112f5636d6280c00dd952ca6b", + "ec34808f3ebd5ba87302edacdc840407eff2c500f10bc9fb823ad8c174d69c14" + ], + "target": "_bin/clangarm64/bin/scalar.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "52ce37aea3c9d6e94391e2df75692dd039e6e782fee53dbca87203574d9cb195": { + "signature": "52ce37aea3c9d6e94391e2df75692dd039e6e782fee53dbca87203574d9cb195", + "alternativeSignatures": [ + "e82ec6e680ff2b96ea3e610377bcf0810398f7f7beb0eb08c56fa2577a38d46c", + "4c61f8c1a7be48bd78362d4fa3e8ffd6cbcacfe888b45929020621a53687cc85", + "8dff1a899beae523beaa2d81402c2622ef3523d848c14a6a6e4637ad468a8401" + ], + "target": "_bin/clangarm64/bin/scalar.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "acd3e4fe55a91ca2a2cc775a6260a3c53d51742aeafa6b60f05418b9f617bcec": { + "signature": "acd3e4fe55a91ca2a2cc775a6260a3c53d51742aeafa6b60f05418b9f617bcec", + "alternativeSignatures": [ + "63ea27de2e6d75ad3086628c9613ca106eb055bda8011d77d85a165d4646d676", + "30c319c8e7addc99a7ea81470bd86b8d7befbbaf9ba534a6b9a347708d830aac", + "0b67f880fd20cec29c300249f7a07428a8ace33b0518c966f1e268237032e68a" + ], + "target": "_bin/clangarm64/libexec/git-core/git-credential-wincred.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "8e142434b17377d3661fe543769ff22883b06878f0dc11b8d753bb22a0ab7924": { + "signature": "8e142434b17377d3661fe543769ff22883b06878f0dc11b8d753bb22a0ab7924", + "alternativeSignatures": [ + "b37bf9cb58efa8200e2777c6b377b08175fcdea07feb64f201e0af09f95b234a", + "062854025b9993575835e305893f958044f9b8841834dd86d2c239ceae0d39a6", + "12478f33504453d75bcdce97a8a9bfa3c7cb227291262178d984a4c269e018fb" + ], + "target": "_bin/clangarm64/libexec/git-core/git-credential-wincred.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "4a536ef6d824cfa87108fa3c69dc1585760bd48e3b914b7ba394b5a7dafb2f6f": { + "signature": "4a536ef6d824cfa87108fa3c69dc1585760bd48e3b914b7ba394b5a7dafb2f6f", + "alternativeSignatures": [ + "72c7ac2cf33af6b48df8798900e05a373f329b7ae3d9e7d14f7eb6019220dec7", + "2e72231910928a8450c5a5da6d0fd48729ffccfb5ce1290e631654da7933b7fc", + "afa2107a5fbb0d85c974f7d2f7ae11905723b555f2005e66a09811aeb097d0f7" + ], + "target": "_bin/clangarm64/libexec/git-core/git-daemon.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "7f397612461002ea0b923bb7d9d5f2de98590dd972da3141d2f6bb41f013f869": { + "signature": "7f397612461002ea0b923bb7d9d5f2de98590dd972da3141d2f6bb41f013f869", + "alternativeSignatures": [ + "0b869312e100f62f2564b684e742b23d24215155967e20e85bdb40f29a9d4a16", + "d0c2382caa251a05c37941631eea371639403627070cca5c5d0f5e1fa9b16ab7", + "4cec7447a7f0186a5db518754a4d5e086670bc45e07a43350b773386c8215803" + ], + "target": "_bin/clangarm64/libexec/git-core/git-daemon.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "6c62fb03e23a0fd656f87a2dab5d4f5fc118d8b3ecc63d1162d3dfbf8fa66908": { + "signature": "6c62fb03e23a0fd656f87a2dab5d4f5fc118d8b3ecc63d1162d3dfbf8fa66908", + "alternativeSignatures": [ + "d93efc939070712fe68165c832fb9cc9464d4557d9610bee28d968ebaba29b0b", + "0c45f7dcc7730fb3d4caf05762cc3e5844868cd4e253673738fea9133704e7ec", + "3555a3206d624f227fdb42598c3bb6381baffd49a584a2d3e07dc17f5b6b10c6" + ], + "target": "_bin/clangarm64/libexec/git-core/git-gvfs-helper.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "42402a8c62fa058dcfa40785fbe92b61db33c7bf887bee60d9882fe16f5fa1aa": { + "signature": "42402a8c62fa058dcfa40785fbe92b61db33c7bf887bee60d9882fe16f5fa1aa", + "alternativeSignatures": [ + "d67caf0ef90e16fdc718b4630966d35d4bafb9367e3cd207e1a52fc4d10c68f6", + "c151c73f0250c31a31050050c82f88b0debc2fd1f6b04a8de20bcf8e1efaf1e9", + "5238c5e56be26e93ef48632b1d78b7ddc25b57da6d5e5017b04246276108628e" + ], + "target": "_bin/clangarm64/libexec/git-core/git-gvfs-helper.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "41822e65f219683beea6281ee1792ae859c2f9dc523a0abd797d5cc47202f3b9": { + "signature": "41822e65f219683beea6281ee1792ae859c2f9dc523a0abd797d5cc47202f3b9", + "alternativeSignatures": [ + "15dd845d28d938ced02edd6c528fad5719996abd4bcde9b66af2c5a6653465fc", + "e8521f3109dcff0bc4c7c9568bbe3b88b455524b564d0b6d33ab481a483f0ec7", + "b5d7cca3fd351e7bf372ea60aa6dfdb08aa9fb40fb4cd31b84b4f0b342f2c701" + ], + "target": "_bin/clangarm64/libexec/git-core/git-http-backend.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "7fb2d81e6a268acfff6295245806d9621693248fa0c7b73a6a1c9c8092d9f1a2": { + "signature": "7fb2d81e6a268acfff6295245806d9621693248fa0c7b73a6a1c9c8092d9f1a2", + "alternativeSignatures": [ + "750a0aafe2eb523610eb62722fcc67f9d5e26a00d0fa0888407d0ae67b0979b0", + "9752a5d1961ff45dc60cb28becf85d92320ce8161aa31332aec763617a1a1f47", + "63db412f6bcb108e3dd7d791a180adcd9ddf25a2a99094548f1fcc597923d844" + ], + "target": "_bin/clangarm64/libexec/git-core/git-http-backend.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "258e9da1208c85046acb20d19d2a44f526b966a95a10a88e9a85aedc199edb5c": { + "signature": "258e9da1208c85046acb20d19d2a44f526b966a95a10a88e9a85aedc199edb5c", + "alternativeSignatures": [ + "d4a9eab8db924bea32fff7588a86b91ea01b762aa0f7d4e4f1b843b549342c74", + "1f9da419ca16b5324e3272b96d73ff7f6781be376c91366af76b379c25a0b5a5", + "0f0ab1d0e7012e59b3c0f8d67ca41073f25282f98d7ccc7796e1d3e0f71d292f" + ], + "target": "_bin/clangarm64/libexec/git-core/git-http-fetch.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "e93b8ae0dce05d3ce9545766f6af630393b05a7254ad37c96909b4c2f084f1f1": { + "signature": "e93b8ae0dce05d3ce9545766f6af630393b05a7254ad37c96909b4c2f084f1f1", + "alternativeSignatures": [ + "5d78bacbe66e048f11927208cf97c23ffa190dcd8d7d3e2758cbcba55caed854", + "7ec3eef60729835440019828f841a12b315df5498c5d24c5a057db008f6006a5", + "6a447230b0b0ee4aa2f38198e4bf3e4ffdf49bd49eab8071b8eaa66103e8025b" + ], + "target": "_bin/clangarm64/libexec/git-core/git-http-fetch.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "64c36ad889b4dbf696257a106292ceb49d0bd7960b1202eea4bdae2c6048f432": { + "signature": "64c36ad889b4dbf696257a106292ceb49d0bd7960b1202eea4bdae2c6048f432", + "alternativeSignatures": [ + "6e2397e04dc5f2d6f27118647c20a0573287e8b3086443620dc05b19df0d78bf", + "00db469d5cceac48c3f11f468f5aa364bb9249610a153c1422f3e89fc0bf07aa", + "8a0a4a1607361edf49e6a4e9614de059ce028bb320bfbe7b51b6627654b9387e" + ], + "target": "_bin/clangarm64/libexec/git-core/git-http-push.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "c4e28ac334d75836eea4aa6d4b7b4fc3f96a9c83cf5d139fcbc0ca9f941dbd2b": { + "signature": "c4e28ac334d75836eea4aa6d4b7b4fc3f96a9c83cf5d139fcbc0ca9f941dbd2b", + "alternativeSignatures": [ + "faea7c1e72031702fbcbbecf4d714ba752685fea4d2af81e90a356c9c1e00b37", + "546d697fb97b9f08dc4aa2492665cd3e04c4de7ee771869014d3a889f771e116", + "76e0c9d86595fc958aef0ef723254c0adf9b139cea3bbbde08a9a96117126bcd" + ], + "target": "_bin/clangarm64/libexec/git-core/git-http-push.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "da56ce9a82c659a02a9a9838d90af16151ab09b039b43fe9e08c037452907b26": { + "signature": "da56ce9a82c659a02a9a9838d90af16151ab09b039b43fe9e08c037452907b26", + "alternativeSignatures": [ + "0f9293ee3def00df42bd9b39efbe3f1892e561f85a4fb52344ba57f80801cbef", + "9529b1d6ce6329b78b7d2e31cb34c9cec31f11dfe737eb6fdc976d7894a92f9d", + "b2ebf8b011810768725764842e0de13fe42649db253235febae455e72623e26f" + ], + "target": "_bin/clangarm64/libexec/git-core/git-imap-send.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "2bca8d4fe377a1b88a1d34b01e665a08919799e74aaa5ff9ca7042f35df9d4c9": { + "signature": "2bca8d4fe377a1b88a1d34b01e665a08919799e74aaa5ff9ca7042f35df9d4c9", + "alternativeSignatures": [ + "0c93c672117bbc6515696d44364687424851d42e1aea69468902746bdabdd94b", + "8ab3c7631630c7c25ebd7ac09ad77c764491567d1f0395789cbec915bdbd650e", + "6533c3f8950e3aa9d69537992bbf9d239582de6128bb0338fdbb4107b281be85" + ], + "target": "_bin/clangarm64/libexec/git-core/git-imap-send.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "bada8bbfe22690cb8a77888073ac9b6f3a892825cc4d011caf354d07881da523": { + "signature": "bada8bbfe22690cb8a77888073ac9b6f3a892825cc4d011caf354d07881da523", + "alternativeSignatures": [ + "b2cb68813a8a9bd2a26fe175bc3372477ec63d45bfe117e89f0ce59869e39a0e", + "31f648ef2ed71d3e2f4f09605aaa834eaba542707f2f3d4d39bf9aea0da728e9", + "58f3eb507bf84ac6ea972fc888102f5c475bc517bb9184cb53e5b480fcf43358" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-ftp.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "8dd70fe3d2ec27c498720e615a64730f0b35b0d1eb3c0689e3e1760f40787f78": { + "signature": "8dd70fe3d2ec27c498720e615a64730f0b35b0d1eb3c0689e3e1760f40787f78", + "alternativeSignatures": [ + "d79769d8e345a48c4fe1fd0ac1fdc24d23e5b1079925938f2686f1e18a51a5ed", + "bc33a550d431cdb38f8daada8b80578e8eef144fa5b45d1bb6644d73b39cf434", + "5104fab6800721116c706ea1314990afc0bf9f7ad82ae23a71ef7bd31e550c6b" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-ftp.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "2e6d5886c50812faf24ecf2f51e32ef6bc87a8f4be7da50209a54e66d44eabce": { + "signature": "2e6d5886c50812faf24ecf2f51e32ef6bc87a8f4be7da50209a54e66d44eabce", + "alternativeSignatures": [ + "f1ae25498c830291256f94869881b10cbe85f31eb27de9e4d7541adf4436dec6", + "ca61481c1ccdff92be857c5c8ca4bc8adc40ec4ab88385172eb414eb3243cd9b", + "61f1a7c4e928b2d13a44e1ee026e0fc19640ee114cc1a078d864d271f7dc10f4" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-ftps.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "3572af217ac0b36f110e224d63005887f887661e56b03f36480ad9167d66b538": { + "signature": "3572af217ac0b36f110e224d63005887f887661e56b03f36480ad9167d66b538", + "alternativeSignatures": [ + "163d70cc710d914d2d78715b15c536eb5e598ea27e50fd1f2dbffddfd013cda6", + "4ef58dba5f8b2806111cf9e42d242719061a6e7426b15fb50d8a46ba837ad889", + "784d2389ce365c46d764caf161755708f48557dbf21d63518f57a689a30b5757" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-ftps.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "da4db908d4714316a7021b7074ede1f140f204fbb0f97db4f709a2781d666d0f": { + "signature": "da4db908d4714316a7021b7074ede1f140f204fbb0f97db4f709a2781d666d0f", + "alternativeSignatures": [ + "2b6976a9262b44ec3a615eccda007d6c3cb17e4aea27b606d9c7c4f69ec6955a", + "1ff78576d09f794a9fe9e763fad9ccc0d2c79621b244c2e1f28837eb2c2e33eb", + "f217d0fa40acb783c06217121e67c32a55b0e4bacc83f6695690099e93677a5f" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-http.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "3c8c45beb7bf6968c666b92aef64bd39954b9724bbf073b596b2c773796aafa6": { + "signature": "3c8c45beb7bf6968c666b92aef64bd39954b9724bbf073b596b2c773796aafa6", + "alternativeSignatures": [ + "5b196245fc05ba15e54c13222a6244b5bf3c74f858d0b6c74e9109331aff5711", + "32c1833014a6ff49748bb21bccfe1850e9a3853d3782f03fdc673660404c6c9d", + "7de42ed077e28792771fdfdec4ab79e97252beef79db1dfcf543a4071d787eef" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-http.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "ddc9bd19e3bb4b3b9a6e9806f75df12c73597706da73b9b7c138ca7677f456ac": { + "signature": "ddc9bd19e3bb4b3b9a6e9806f75df12c73597706da73b9b7c138ca7677f456ac", + "alternativeSignatures": [ + "3f768e3cae871508d658bbfa6e20ceb20bd015765a27bc194c0ece44d7e063f3", + "ca10a085cb18a5fc8efbfe56eb0eb741ad318d56e1678544260bbd4d8d57bd92", + "05890a9c899b3730fdc113a0a33e2d7bb6ff4f8a1a729b558e24747431c024ab" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-https.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "8616bda42bc8cbbae07c6ee06825554f4ce75241adf6b9dda910f83f5be748a2": { + "signature": "8616bda42bc8cbbae07c6ee06825554f4ce75241adf6b9dda910f83f5be748a2", + "alternativeSignatures": [ + "eef6c183bc43c1710ba0e441a6f9abcfac5f4def9ec610e3b55946d6eb4138d0", + "3deca0ff646fdb7906c582e3b56ce88c038b04ebb182d90fa57113032308137c", + "12a777ac57bf3927c5a2402ede3389b2f915d6fd36f45d0327d9e5f43f1c267d" + ], + "target": "_bin/clangarm64/libexec/git-core/git-remote-https.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "2b8c2d6fa2b4a0f3fba6a4fc872e49218752e9f856d8e8a1523caf28dcf6ac84": { + "signature": "2b8c2d6fa2b4a0f3fba6a4fc872e49218752e9f856d8e8a1523caf28dcf6ac84", + "alternativeSignatures": [ + "9457d3dbc084738158db9b44e3105f1b04190566b4d45e1042064a31ad5a784c", + "b4f364d0c2ecc39343c22d3cbf01d4ec4b16db7e2b28bf2e8d10cb5a12d5f8eb", + "11e4030456e9b5c6cbfec13252c8b9f60cdd96c3818165f860704a514698d118" + ], + "target": "_bin/clangarm64/libexec/git-core/git-sh-i18n--envsubst.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "2e20ced8265c12dfc4a6775e9492ec7a3c6de4a29f8a8b2ca6495cc9d5aa1a43": { + "signature": "2e20ced8265c12dfc4a6775e9492ec7a3c6de4a29f8a8b2ca6495cc9d5aa1a43", + "alternativeSignatures": [ + "4f25ba6b45b3354db39c8c2473c96ae732794298a976c349a3396ed101e86fe2", + "d0babd76daa2826b3f379c5461730ecd1264204b7459fc87d8630542d04a1307", + "0e5e274353ec1cb59b7d234586921655153480ed8dfbffed02d069a27ed52467" + ], + "target": "_bin/clangarm64/libexec/git-core/git-sh-i18n--envsubst.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "0e6ae34b65bf6003767d3b71220988fb0d17c8d48531752957ff6f1278a3bff1": { + "signature": "0e6ae34b65bf6003767d3b71220988fb0d17c8d48531752957ff6f1278a3bff1", + "alternativeSignatures": [ + "a6c867253954137604add56197f70f64ec0cf9e1cab72044293882b553f706c7", + "26b51c0853c53721f4e1dc04e77cf70b7bc20018014686c8ba7430bc3939a040", + "9eafd480a66e6b3d256855b44ee238ea7de974353c8b3cfd03a543f19402c7f9" + ], + "target": "_bin/clangarm64/libexec/git-core/git-shell.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "597c2243d3c8d0b732c8ac676f9221f5590f0db0223d63ff842a4649bae93655": { + "signature": "597c2243d3c8d0b732c8ac676f9221f5590f0db0223d63ff842a4649bae93655", + "alternativeSignatures": [ + "691b1d7082c4dd80f54cd2aa83c0f1e353c9b28c654e8fce4e865634edc480e1", + "ad442e559f3d14514b6ed4325d9febd171172ca1a165cbfb50fbde9f42ae604e", + "6481d4fa4b09642ac72ce891c36a4096501fca0d7ff458f79250fc47e692f2f2" + ], + "target": "_bin/clangarm64/libexec/git-core/git-shell.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "212a23b0f5998525edba472040b36694195c82022ddfdc79d47b02c1830f236b": { + "signature": "212a23b0f5998525edba472040b36694195c82022ddfdc79d47b02c1830f236b", + "alternativeSignatures": [ + "77afe7e6fe80b801e99256c2eddc859dfa27ac5372b1f0864624d5cebcd0c933", + "e60f8f431f72838a7b35578a6b6cff7dbd28604072fe5dad64b9034308b6314f", + "291981ab4722761a9a2d72edb529b5ebd83ed7fd23ce15771a203c8eecaf4a52" + ], + "target": "_bin/clangarm64/libexec/git-core/git.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "418b7e1a1643a9987984872840ad7af4c6232beab5a80c5a25bf4fb9ca9b575c": { + "signature": "418b7e1a1643a9987984872840ad7af4c6232beab5a80c5a25bf4fb9ca9b575c", + "alternativeSignatures": [ + "6871337b645c59668a770bd89cb6900382bb231baa21d8b80012cab646667e32", + "f3317f1bae546eec8e8e3d47b820e4c14aea68246a114a9465efede36e190d6f", + "e0fc0f393dc7cb3cd69c6fcf7b515d439769a73ab265b30aa9ec2444fb7127f6" + ], + "target": "_bin/clangarm64/libexec/git-core/git.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "08f7b3d2a618c656bae10a7600a869667cf13d28f0e7a200d172974676ba57e0": { + "signature": "08f7b3d2a618c656bae10a7600a869667cf13d28f0e7a200d172974676ba57e0", + "alternativeSignatures": [ + "799bd7f48384fa663ab8c7c4c56ca1ea237543a246c121461981de648c72a47b", + "e83948a022890dfdaa78dec614d634078a9121b5d5f76bb23f418d62f568894d", + "af845195e9b490c5347eeb567bcd4d72d400929b2511fee8c346b0c93cb63b72" + ], + "target": "_bin/clangarm64/libexec/git-core/headless-git.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "aed97c6e942ece3edd725fcc89d444de6e84435964037ca6b5575a95e7464132": { + "signature": "aed97c6e942ece3edd725fcc89d444de6e84435964037ca6b5575a95e7464132", + "alternativeSignatures": [ + "c9b1cecf6c51bdca4625ad89f24453afb0e2ad95edaa6cebf433f3e10bb044a8", + "4786a5b866faa860a535d45ab7d5f372907bd94f67e1573b332a5bd543eb023c", + "9918a8269de76fc32e119f45c6b0e79d83da873596da4cfae9f2e8458b47f3ed" + ], + "target": "_bin/clangarm64/libexec/git-core/headless-git.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + }, + "18ea9aeec2f6a54008bed03c5390a832fc1ae1d7abd398698160a956f375ddf1": { + "signature": "18ea9aeec2f6a54008bed03c5390a832fc1ae1d7abd398698160a956f375ddf1", + "alternativeSignatures": [ + "34de9fd6a96e87bf9a94c588c10f3218853b25354693278cfbe88ceb2dbdeb9e", + "ea7e0b5b236c03737be9e93bf18e1d2f1db2b4e5c3f26d2f940d542b7e9c815c", + "9ba8e3e4850fa3cb31deebf61aa7cea2c8b3bf42f8b8780cf2ef0a0e05c90f93" + ], + "target": "_bin/clangarm64/libexec/git-core/scalar.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2008", + "createdDate": "2026-05-27 10:14:26Z" + }, + "2ae14339623ae3e433dd3eed27578564ad7a6b2757cc3ee937a89d49a65617bc": { + "signature": "2ae14339623ae3e433dd3eed27578564ad7a6b2757cc3ee937a89d49a65617bc", + "alternativeSignatures": [ + "9bf994df30dfd0ca3e278381e18bef22c76d33c50153ebe2e367882fcb911c10", + "e9b29b67a2c81b41641ad1f2888b2b8db0b6ee92755f3299411cdcd4a5e50582", + "b672c20584f44e68c4915f686b0a69a8121002d775173e9d0c08cdc692119318" + ], + "target": "_bin/clangarm64/libexec/git-core/scalar.exe", + "uriBaseId": "file:///D:/a/_work/1/a/", + "memberOf": [ + "default" + ], + "tool": "binskim", + "ruleId": "BA2012", + "createdDate": "2026-05-27 10:14:26Z" + } + } +} \ No newline at end of file diff --git a/.azure-pipelines/sdl/windows_x64/.gdnsuppress b/.azure-pipelines/sdl/windows_x64/.gdnsuppress new file mode 100644 index 00000000000000..9c7262a29972f5 --- /dev/null +++ b/.azure-pipelines/sdl/windows_x64/.gdnsuppress @@ -0,0 +1,15 @@ +{ + "hydrated": true, + "properties": { + "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/suppressions" + }, + "version": "1.0.0", + "suppressionSets": { + "default": { + "name": "default", + "createdDate": "2026-05-28 11:00:00Z", + "lastUpdatedDate": "2026-05-28 11:00:00Z" + } + }, + "results": {} +} diff --git a/.config/.gitattributes b/.config/.gitattributes new file mode 100644 index 00000000000000..cbbb05c1b208e7 --- /dev/null +++ b/.config/.gitattributes @@ -0,0 +1 @@ +* whitespace=-trail,-space,-incomplete diff --git a/.config/1espt/PipelineAutobaseliningConfig.yml b/.config/1espt/PipelineAutobaseliningConfig.yml new file mode 100644 index 00000000000000..ef1a875808578a --- /dev/null +++ b/.config/1espt/PipelineAutobaseliningConfig.yml @@ -0,0 +1,19 @@ +## DO NOT MODIFY THIS FILE MANUALLY. This is part of auto-baselining from 1ES Pipeline Templates. Go to [https://aka.ms/1espt-autobaselining] for more details. + +pipelines: + 22503: + retail: + source: + eslint: + lastModifiedDate: 2026-05-29 + psscriptanalyzer: + lastModifiedDate: 2026-05-29 + armory: + lastModifiedDate: 2026-05-29 + accessibilityinsights: + lastModifiedDate: 2026-05-29 + binary: + binskim: + lastModifiedDate: 2026-05-29 + spotbugs: + lastModifiedDate: 2026-05-29 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000000000..c19530b086311a --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,68 @@ + - [ ] I was not able to find an [open](https://github.com/microsoft/git/issues?q=is%3Aopen) + or [closed](https://github.com/microsoft/git/issues?q=is%3Aclosed) issue matching + what I'm seeing, including in [the `git-for-windows/git` tracker](https://github.com/git-for-windows/git/issues). + +### Setup + + - Which version of `microsoft/git` are you using? Is it 32-bit or 64-bit? + +``` +$ git --version --build-options + +** insert your machine's response here ** +``` + +Are you using Scalar or VFS for Git? + +** insert your answer here ** + +If VFS for Git, then what version? + +``` +$ gvfs version + +** insert your machine's response here ** +``` + + - Which version of Windows are you running? Vista, 7, 8, 10? Is it 32-bit or 64-bit? + +``` +$ cmd.exe /c ver + +** insert your machine's response here ** +``` + + - Any other interesting things about your environment that might be related + to the issue you're seeing? + +** insert your response here ** + +### Details + + - Which terminal/shell are you running Git from? e.g Bash/CMD/PowerShell/other + +** insert your response here ** + + - What commands did you run to trigger this issue? If you can provide a + [Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) + this will help us understand the issue. + +``` +** insert your commands here ** +``` + - What did you expect to occur after running these commands? + +** insert here ** + + - What actually happened instead? + +** insert here ** + + - If the problem was occurring with a specific repository, can you specify + the repository? + + * [ ] Public repo: **insert URL here** + * [ ] Windows monorepo + * [ ] Office monorepo + * [ ] Other Microsoft-internal repo: **insert name here** + * [ ] Other internal repo. diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml deleted file mode 100644 index b49593339932b2..00000000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Bug report -description: Use this template to report bugs. -body: - - type: checkboxes - id: search - attributes: - label: Existing issues matching what you're seeing - description: Please search for [open](https://github.com/git-for-windows/git/issues?q=is%3Aopen) or [closed](https://github.com/git-for-windows/git/issues?q=is%3Aclosed) issue matching what you're seeing before submitting a new issue. - options: - - label: I was not able to find an open or closed issue matching what I'm seeing - - type: textarea - id: git-for-windows-version - attributes: - label: Git for Windows version - description: Which version of Git for Windows are you using? - placeholder: Please insert the output of `git --version --build-options` here - render: shell - validations: - required: true - - type: dropdown - id: windows-version - attributes: - label: Windows version - description: Which version of Windows are you running? - options: - - Windows 8.1 - - Windows 10 - - Windows 11 - - Other - default: 2 - validations: - required: true - - type: dropdown - id: windows-arch - attributes: - label: Windows CPU architecture - description: What CPU Archtitecture does your Windows target? - options: - - i686 (32-bit) - - x86_64 (64-bit) - - ARM64 - default: 1 - validations: - required: true - - type: textarea - id: windows-version-cmd - attributes: - label: Additional Windows version information - description: This provides us with further information about your Windows such as the build number - placeholder: Please insert the output of `cmd.exe /c ver` here - render: shell - - type: textarea - id: options - attributes: - label: Options set during installation - description: What options did you set as part of the installation? Or did you choose the defaults? - placeholder: | - One of the following: - > type "C:\Program Files\Git\etc\install-options.txt" - > type "C:\Program Files (x86)\Git\etc\install-options.txt" - > type "%USERPROFILE%\AppData\Local\Programs\Git\etc\install-options.txt" - > type "$env:USERPROFILE\AppData\Local\Programs\Git\etc\install-options.txt" - $ cat /etc/install-options.txt - render: shell - validations: - required: true - - type: textarea - id: other-things - attributes: - label: Other interesting things - description: Any other interesting things about your environment that might be related to the issue you're seeing? - - type: input - id: terminal - attributes: - label: Terminal/shell - description: Which terminal/shell are you running Git from? e.g Bash/CMD/PowerShell/other - validations: - required: true - - type: textarea - id: commands - attributes: - label: Commands that trigger the issue - description: What commands did you run to trigger this issue? If you can provide a [Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) this will help us understand the issue. - render: shell - validations: - required: true - - type: textarea - id: expected-behaviour - attributes: - label: Expected behaviour - description: What did you expect to occur after running these commands? - validations: - required: true - - type: textarea - id: actual-behaviour - attributes: - label: Actual behaviour - description: What actually happened instead? - validations: - required: true - - type: textarea - id: repository - attributes: - label: Repository - description: If the problem was occurring with a specific repository, can you provide the URL to that repository to help us with testing? \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index ec4bb386bcf8a4..00000000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1 +0,0 @@ -blank_issues_enabled: false \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7baf31f2c471ec..3cb48d8582f31c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,22 +1,10 @@ Thanks for taking the time to contribute to Git! -Those seeking to contribute to the Git for Windows fork should see -http://gitforwindows.org/#contribute on how to contribute Windows specific -enhancements. - -If your contribution is for the core Git functions and documentation -please be aware that the Git community does not use the github.com issues -or pull request mechanism for their contributions. - -Instead, we use the Git mailing list (git@vger.kernel.org) for code and -documentation submissions, code reviews, and bug reports. The -mailing list is plain text only (anything with HTML is sent directly -to the spam folder). - -Nevertheless, you can use GitGitGadget (https://gitgitgadget.github.io/) -to conveniently send your Pull Requests commits to our mailing list. - -For a single-commit pull request, please *leave the pull request description -empty*: your commit message itself should describe your changes. - -Please read the "guidelines for contributing" linked above! +This fork contains changes specific to monorepo scenarios. If you are an +external contributor, then please detail your reason for submitting to +this fork: + +* [ ] This is an early version of work already under review upstream. +* [ ] This change only applies to interactions with Azure DevOps and the + GVFS Protocol. +* [ ] This change only applies to the virtualization hook and VFS for Git. diff --git a/.github/actions/akv-secret/action.yml b/.github/actions/akv-secret/action.yml new file mode 100644 index 00000000000000..50412405e89e2c --- /dev/null +++ b/.github/actions/akv-secret/action.yml @@ -0,0 +1,54 @@ +name: Get Azure Key Vault Secrets + +description: | + Get secrets from Azure Key Vault and store the results as masked step outputs, + environment variables, or files. + +inputs: + vault: + required: true + description: Name of the Azure Key Vault. + secrets: + required: true + description: | + Comma- or newline-separated list of secret names in Azure Key Vault. + The output and encoding of secrets can be specified using this syntax: + + SECRET ENCODING> $output:OUTPUT + SECRET ENCODING> $env:ENVAR + SECRET ENCODING> FILE + + SECRET Name of the secret in Azure Key Vault. + ENCODING (optional) Encoding of the secret: base64. + OUTPUT Name of a step output variable. + ENVAR Name of an environment variable. + FILE File path (relative or absolute). + + If no output format is specified the default is a step output variable + with the same name as the secret. I.e, SECRET > $output:SECRET. + + Examples: + + Assign output variable named `raw-var` to the raw value of the secret + `raw-secret`: + + raw-secret > $output:raw-var + + Assign output variable named `decoded-var` to the base64 decoded value + of the secret `encoded-secret`: + + encoded-secret base64> $output:decoded-var + + Download the secret named `tls-certificate` to the file path + `.certs/tls.cert`: + + tls-certificate > .certs/tls.cert + + Assign environment variable `ENV_SECRET` to the base64 decoded value of + the secret `encoded-secret`: + + encoded-secret base64> $env:ENV_SECRET + +runs: + using: node24 + main: index.js diff --git a/.github/actions/akv-secret/index.js b/.github/actions/akv-secret/index.js new file mode 100644 index 00000000000000..19a930db983b0c --- /dev/null +++ b/.github/actions/akv-secret/index.js @@ -0,0 +1,195 @@ +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { isUtf8 } = require("buffer"); + +// Note that we are not using the `@actions/core` package as it is not available +// without either committing node_modules/ to the repository, or using something +// like ncc to bundle the code. + +// See https://github.com/actions/toolkit/blob/%40actions/core%401.1.0/packages/core/src/command.ts#L81-L87 +const escapeData = (s) => { + return s + .replace(/%/g, '%25') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A') +} + +const stringify = (value) => { + if (typeof value === 'string') return value; + if (Buffer.isBuffer(value) && isUtf8(value)) return value.toString('utf-8'); + return undefined; +} + +const trimEOL = (buf) => { + let l = buf.length + if (l > 0 && buf[l - 1] === 0x0a) { + l -= l > 1 && buf[l - 2] === 0x0d ? 2 : 1 + } + return buf.slice(0, l) +} + +const writeBufToFile = (buf, file) => { + out = fs.createWriteStream(file) + out.write(buf) + out.end() +} + +const logInfo = (message) => { + process.stdout.write(`${message}${os.EOL}`); +} + +const setFailed = (error) => { + process.stdout.write(`::error::${escapeData(error.message)}${os.EOL}`); + process.exitCode = 1; +} + +const writeCommand = (file, name, value) => { + // Unique delimiter to avoid conflicts with actual values + let delim; + for (let count = 0; ; count++) { + delim = `XXXXXX${count}`; + if (!name.includes(delim) && !value.includes(delim)) { + break; + } + } + + fs.appendFileSync(file, `${name}<<${delim}${os.EOL}${value}${os.EOL}${delim}${os.EOL}`); +} + +const setSecret = (value) => { + value = stringify(value); + + // Masking a secret that is not a valid UTF-8 string or buffer is not useful + if (value === undefined) return; + + process.stdout.write( + value + .split(/\r?\n/g) + .filter(line => line.length > 0) // Cannot mask empty lines + .map( + value => `::add-mask::${escapeData(value)}${os.EOL}` + ) + .join('') + ); +} + +const setOutput = (name, value) => { + value = stringify(value); + if (value === undefined) { + throw new Error(`Output value '${name}' is not a valid UTF-8 string or buffer`); + } + + writeCommand(process.env.GITHUB_OUTPUT, name, value); +} + +const exportVariable = (name, value) => { + value = stringify(value); + if (value === undefined) { + throw new Error(`Environment variable '${name}' is not a valid UTF-8 string or buffer`); + } + + writeCommand(process.env.GITHUB_ENV, name, value); +} + +(async () => { + const vault = process.env.INPUT_VAULT; + const secrets = process.env.INPUT_SECRETS; + // Parse and normalize secret mappings + const secretMappings = secrets + .split(/[\n,]+/) + .map((entry) => entry.trim()) + .filter((entry) => entry) + .map((entry) => { + const [input, encoding, output] = entry.split(/(\S+)?>/).map((part) => part?.trim()); + return { input, encoding, output: output || `\$output:${input}` }; // Default output to $output:input if not specified + }); + + if (secretMappings.length === 0) { + throw new Error('No secrets provided.'); + } + + // Fetch secrets from Azure Key Vault + for (const { input: secretName, encoding, output } of secretMappings) { + let az = spawnSync('az', + [ + 'keyvault', + 'secret', + 'show', + '--vault-name', + vault, + '--name', + secretName, + '--query', + 'value', + '--output', + 'tsv' + ], + { + stdio: ['ignore', 'pipe', 'inherit'], + shell: true // az is a batch script on Windows + } + ); + + if (az.error) throw new Error(az.error, { cause: az.error }); + if (az.status !== 0) throw new Error(`az failed with status ${az.status}`); + + // az keyvault secret show --output tsv returns a buffer with trailing \n + // (or \r\n on Windows), so we need to trim it specifically. + let secretBuf = trimEOL(az.stdout); + + // Mask the raw secret value in logs + setSecret(secretBuf); + + // Handle encoded values if specified + // Sadly we cannot use the `--encoding` parameter of the `az keyvault + // secret (show|download)` command as the former does not support it, and + // the latter must be used with `--file` (we could use /dev/stdout on UNIX + // but not on Windows). + if (encoding) { + switch (encoding.toLowerCase()) { + case 'base64': + secretBuf = Buffer.from(secretBuf.toString('utf-8'), 'base64'); + break; + case 'ascii': + case 'utf8': + case 'utf-8': + // No need to decode the existing buffer from the az command + break; + default: + throw new Error(`Unsupported encoding: ${encoding}`); + } + + // Mask the decoded value + setSecret(secretBuf); + } + + const outputType = output.startsWith('$env:') + ? 'env' + : output.startsWith('$output:') + ? 'output' + : 'file'; + + switch (outputType) { + case 'env': + const varName = output.replace('$env:', '').trim(); + exportVariable(varName, secretBuf); + logInfo(`Secret set as environment variable: ${varName}`); + break; + + case 'output': + const outputName = output.replace('$output:', '').trim(); + setOutput(outputName, secretBuf); + logInfo(`Secret set as output variable: ${outputName}`); + break; + + case 'file': + const filePath = output.trim(); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + writeBufToFile(secretBuf, filePath); + logInfo(`Secret written to file: ${filePath}`); + break; + } + } +})().catch(setFailed); diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 00000000000000..d9ae1e8487a09d --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,134 @@ +name: "CodeQL config" + +queries: + - uses: security-extended + +paths-ignore: + - gitweb/**/*.js # GitWeb is not distributed + +query-filters: + - exclude: + # yes, this extra indentation is intentional + # too common in Git's source code + id: cpp/trivial-switch + - exclude: + id: cpp/loop-variable-changed + - exclude: + # we override this locally with a modified version + id: cpp/non-constant-format + - exclude: + # Git does not consider this a problem + id: cpp/irregular-enum-init + - exclude: + # Git has many long functions, this alert would match too many + id: cpp/poorly-documented-function + - exclude: + # In Git, there is a lot of commented-out code + id: cpp/commented-out-code + - exclude: + # While it is true that long switch cases are hard to read and + # validate, Git has way too many for us to allow this query to + # churn out alerts left and right + id: cpp/long-switch + - exclude: + # CodeQL does not expect Git to heed the umask(), but it does + id: cpp/world-writable-file-creation + - exclude: + # Git uses the construct `if () ; else ...` often, to + # avoid an extra indentation level. CodeQL does not like that. + id: cpp/empty-block + - exclude: + # This rule unfortunately triggers some false positives, e.g. + # where Git tries to redact URLs or where Git specifically + # asks for a password upon GIT_SSL_CERT_PASSWORD_PROTECTED. + id: cpp/user-controlled-bypass + - exclude: + # This rule fails to recognize that xmallocz() _specifically_ + # makes room for a trailing NUL, and instead assumes that this + # function behaves like malloc(), which does not. + id: cpp/invalid-pointer-deref + - exclude: + # CodeQL fails to recognize that xmallocz() accounts for the NUL, + # instead assuming malloc() semantics. + id: cpp/no-space-for-terminator + - exclude: + # Git does exchange plain-text passwords via stdin/stdout e.g. + # with helpers in the credential protocol, or in credential-cache. + # This rule, though, assumes that writing to _any_ file descriptor + # is unsafe. + id: cpp/cleartext-storage-file + - exclude: + # When storing the value of the environment variable `PWD` as the + # current directory in absolute_pathdup(), or when allocating memory + # for a binary patch where the size is specified in the patch itself, + # CodeQL assumes that this can lead to a denial of service because + # of an unbounded size, but Git's code works as designed here. + id: cpp/uncontrolled-allocation-size + - exclude: + # lock_repo_for_gc() has admittedly obtuse logic to parse the + # process ID out of the `gc.pid` file, which is correct, but + # due to its construction throws a false positive here. + id: cpp/missing-check-scanf + - exclude: + # discard_cache_entry() overwrites the name in a FLEX_ARRAY struct + # if GIT_TEST_VALIDATE_INDEX_CACHE_ENTRIES is set, which CodeQL fails + # to recognize as valid. + id: cpp/overrun-write + - exclude: + # Since `time_t` can be signed or unsigned, there is unfortunately + # no way to avoid letting this rule report a potential + id: cpp/integer-multiplication-cast-to-long + - exclude: + # There are many, many legitimate code paths in Git where a path is + # constructed from an environment variable, e.g. GIT_DIR. Let's suppress + # this slightly overzealous query. + id: cpp/path-injection + - exclude: + # Git has 99 instances of this at the time of writing :-( + id: cpp/declaration-hides-variable + - exclude: + id: cpp/declaration-hides-parameter + - exclude: + id: cpp/local-variable-hides-global-variable + - exclude: + id: cpp/complex-condition + - exclude: + # Nested, long-winded switch statements are hard to read and hard + # to reason about. Looking at you, `format_commit_one()`. + id: cpp/complex-block + - exclude: + # There are four instances of this at time of writing, all intentional. + # However, it is very easy to introduce unintentional re-use of loop + # variable names, therefore we will most likely want to either change these + # instances or add suppressions. + id: cpp/nested-loops-with-same-variable + - exclude: + # zOMG so many FIXMEs + id: cpp/fixme-comment + - exclude: + # Git assumes quite a bit about the user's control of the current worktree + # Therefore, it kind of assumes that TOCTOU issues are not a thing when + # it comes to files. + id: cpp/toctou-race-condition + - exclude: + # Too many results in Git where the code was, however, intentionally written + # the way it is. + id: cpp/stack-address-escape + - exclude: + id: cpp/inconsistent-null-check + - exclude: + # This would trigger alerts in the functions in `help.c` that want to open + # external programs to show manual pages. + id: cpp/uncontrolled-process-operation + - exclude: + # The code in t/unit-tests/u-ctype.c implicitly exercises the `sane_istest()` + # macro extensively, and CodeQL seems to miss the cast to `(unsigned char)`, + # thereby mistaking the accesses for being past the end of the array (which + # is incorrect). + # + # Ideally, we would exclude test programs from CodeQL anyways, but + # unfortunately there is no Makefile rule in Git's code base to build only + # the production code, and CodeQL's `paths-ignore` directive described at + # https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#specifying-directories-to-scan + # unfortunately is _ignored_ for compiled languages. + id: cpp/overflow-buffer diff --git a/.github/macos-installer/Makefile b/.github/macos-installer/Makefile new file mode 100644 index 00000000000000..949995b08e6291 --- /dev/null +++ b/.github/macos-installer/Makefile @@ -0,0 +1,164 @@ +SHELL := /bin/bash +SUDO := sudo +C_INCLUDE_PATH := /usr/include +CPLUS_INCLUDE_PATH := /usr/include +LD_LIBRARY_PATH := /usr/lib + +OSX_VERSION := $(shell sw_vers -productVersion) +TARGET_FLAGS := -mmacosx-version-min=$(OSX_VERSION) -DMACOSX_DEPLOYMENT_TARGET=$(OSX_VERSION) + +uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') + +ARCH_UNIV := universal +ARCH_FLAGS := -arch x86_64 -arch arm64 + +CFLAGS := $(TARGET_FLAGS) $(ARCH_FLAGS) +LDFLAGS := $(TARGET_FLAGS) $(ARCH_FLAGS) + +PREFIX := /usr/local +GIT_PREFIX := $(PREFIX)/git + +# Replace -rc with .rc in the version string +# This is to ensure compatibility with the format as generated by GIT-VERSION-GEN +ORIGINAL_VERSION := $(VERSION) +# VERSION := $(subst -rc,.rc,$(VERSION)) + +BUILD_DIR := $(GITHUB_WORKSPACE)/payload +DESTDIR := $(PWD)/stage/git-$(ARCH_UNIV)-$(VERSION) +ARTIFACTDIR := build-artifacts +SUBMAKE := $(MAKE) C_INCLUDE_PATH="$(C_INCLUDE_PATH)" CPLUS_INCLUDE_PATH="$(CPLUS_INCLUDE_PATH)" LD_LIBRARY_PATH="$(LD_LIBRARY_PATH)" TARGET_FLAGS="$(TARGET_FLAGS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" NO_GETTEXT=1 NO_DARWIN_PORTS=1 prefix=$(GIT_PREFIX) GIT_BUILT_FROM_COMMIT="$(GIT_BUILT_FROM_COMMIT)" DESTDIR=$(DESTDIR) +CORES := $(shell bash -c "sysctl hw.ncpu | awk '{print \$$2}'") + +# Guard against environment variables +APPLE_APP_IDENTITY = +APPLE_INSTALLER_IDENTITY = +APPLE_KEYCHAIN_PROFILE = + +.PHONY: image pkg payload codesign notarize + +.SECONDARY: + +$(DESTDIR)$(GIT_PREFIX)/VERSION-$(VERSION)-$(ARCH_UNIV): + rm -f $(BUILD_DIR)/git-$(VERSION)/osx-installed* + mkdir -p $(DESTDIR)$(GIT_PREFIX) + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-built-keychain: + cd $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain; $(SUBMAKE) CFLAGS="$(CFLAGS) -g -O2 -Wall" + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-built: + [ -d $(DESTDIR)$(GIT_PREFIX) ] && $(SUDO) rm -rf $(DESTDIR) || echo ok + cd $(BUILD_DIR)/git-$(VERSION); $(SUBMAKE) -j $(CORES) all strip + echo "================" + echo "Dumping Linkage" + cd $(BUILD_DIR)/git-$(VERSION); ./git version + echo "====" + cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git + echo "====" + cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-http-fetch + echo "====" + cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-http-push + echo "====" + cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-remote-http + echo "====" + cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-gvfs-helper + echo "================" + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-installed-bin: $(BUILD_DIR)/git-$(VERSION)/osx-built $(BUILD_DIR)/git-$(VERSION)/osx-built-keychain + cd $(BUILD_DIR)/git-$(VERSION); $(SUBMAKE) install + cp $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain/git-credential-osxkeychain $(DESTDIR)$(GIT_PREFIX)/bin/git-credential-osxkeychain + mkdir -p $(DESTDIR)$(GIT_PREFIX)/contrib/completion + cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-completion.bash $(DESTDIR)$(GIT_PREFIX)/contrib/completion/ + cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-completion.zsh $(DESTDIR)$(GIT_PREFIX)/contrib/completion/ + cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-prompt.sh $(DESTDIR)$(GIT_PREFIX)/contrib/completion/ + # This is needed for Git-Gui, GitK + mkdir -p $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl + [ ! -f $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl/Error.pm ] && cp $(BUILD_DIR)/git-$(VERSION)/perl/private-Error.pm $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl/Error.pm || echo done + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-installed-man: $(BUILD_DIR)/git-$(VERSION)/osx-installed-bin + mkdir -p $(DESTDIR)$(GIT_PREFIX)/share/man + cp -R $(GITHUB_WORKSPACE)/manpages/ $(DESTDIR)$(GIT_PREFIX)/share/man + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-built-subtree: + $(SUBMAKE) -C $(BUILD_DIR)/git-$(VERSION)/Documentation asciidoc.conf + cd $(BUILD_DIR)/git-$(VERSION)/contrib/subtree; $(SUBMAKE) XML_CATALOG_FILES="$(XML_CATALOG_FILES)" all git-subtree.1 + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-installed-subtree: $(BUILD_DIR)/git-$(VERSION)/osx-built-subtree + mkdir -p $(DESTDIR) + cd $(BUILD_DIR)/git-$(VERSION)/contrib/subtree; $(SUBMAKE) XML_CATALOG_FILES="$(XML_CATALOG_FILES)" install install-man + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-installed-assets: $(BUILD_DIR)/git-$(VERSION)/osx-installed-bin + mkdir -p $(DESTDIR)$(GIT_PREFIX)/etc + cat assets/etc/gitconfig.osxkeychain >> $(DESTDIR)$(GIT_PREFIX)/etc/gitconfig + cp assets/uninstall.sh $(DESTDIR)$(GIT_PREFIX)/uninstall.sh + sh -c "echo .DS_Store >> $(DESTDIR)$(GIT_PREFIX)/share/git-core/templates/info/exclude" + +symlinks: + mkdir -p $(ARTIFACTDIR)$(PREFIX)/bin + cd $(ARTIFACTDIR)$(PREFIX)/bin; find ../git/bin -type f -exec ln -sf {} \; + for man in man1 man3 man5 man7; do mkdir -p $(ARTIFACTDIR)$(PREFIX)/share/man/$$man; (cd $(ARTIFACTDIR)$(PREFIX)/share/man/$$man; ln -sf ../../../git/share/man/$$man/* ./); done + ruby ../scripts/symlink-git-hardlinks.rb $(ARTIFACTDIR) + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-installed: $(DESTDIR)$(GIT_PREFIX)/VERSION-$(VERSION)-$(ARCH_UNIV) $(BUILD_DIR)/git-$(VERSION)/osx-installed-man $(BUILD_DIR)/git-$(VERSION)/osx-installed-assets $(BUILD_DIR)/git-$(VERSION)/osx-installed-subtree + find $(DESTDIR)$(GIT_PREFIX) -type d -exec chmod ugo+rx {} \; + find $(DESTDIR)$(GIT_PREFIX) -type f -exec chmod ugo+r {} \; + touch $@ + +$(BUILD_DIR)/git-$(VERSION)/osx-built-assert-$(ARCH_UNIV): $(BUILD_DIR)/git-$(VERSION)/osx-built + File $(BUILD_DIR)/git-$(VERSION)/git + File $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain/git-credential-osxkeychain + touch $@ + +disk-image/VERSION-$(VERSION)-$(ARCH_UNIV): + rm -f disk-image/*.pkg disk-image/VERSION-* disk-image/.DS_Store + mkdir disk-image + touch "$@" + +pkg_cmd := pkgbuild --identifier com.git.pkg --version $(VERSION) \ + --root $(ARTIFACTDIR)$(PREFIX) --scripts assets/scripts \ + --install-location $(PREFIX) + +ifdef APPLE_INSTALLER_IDENTITY + pkg_cmd += --sign "$(APPLE_INSTALLER_IDENTITY)" +endif + +pkg_cmd += disk-image/git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).pkg + +disk-image/git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).pkg: disk-image/VERSION-$(VERSION)-$(ARCH_UNIV) symlinks + $(pkg_cmd) + +git-%-$(ARCH_UNIV).dmg: + hdiutil create git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).uncompressed.dmg -fs HFS+ -srcfolder disk-image -volname "Git $(ORIGINAL_VERSION) $(ARCH_UNIV)" -ov 2>&1 | tee err || { \ + grep "Resource busy" err && \ + sleep 5 && \ + hdiutil create git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).uncompressed.dmg -fs HFS+ -srcfolder disk-image -volname "Git $(ORIGINAL_VERSION) $(ARCH_UNIV)" -ov; } + hdiutil convert -format UDZO -o $@ git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).uncompressed.dmg + rm -f git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).uncompressed.dmg + +payload: $(BUILD_DIR)/git-$(VERSION)/osx-installed $(BUILD_DIR)/git-$(VERSION)/osx-built-assert-$(ARCH_UNIV) + +pkg: disk-image/git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).pkg + +image: git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).dmg + +ifdef APPLE_APP_IDENTITY +codesign: + @$(CURDIR)/../scripts/codesign.sh --payload="build-artifacts/usr/local/git" \ + --identity="$(APPLE_APP_IDENTITY)" \ + --entitlements="$(CURDIR)/entitlements.xml" +endif + +# Notarization can only happen if the package is fully signed +ifdef APPLE_KEYCHAIN_PROFILE +notarize: + @$(CURDIR)/../scripts/notarize.sh \ + --package="disk-image/git-$(ORIGINAL_VERSION)-$(ARCH_UNIV).pkg" \ + --keychain-profile="$(APPLE_KEYCHAIN_PROFILE)" +endif diff --git a/.github/macos-installer/assets/etc/gitconfig.osxkeychain b/.github/macos-installer/assets/etc/gitconfig.osxkeychain new file mode 100644 index 00000000000000..788266b3a40a9d --- /dev/null +++ b/.github/macos-installer/assets/etc/gitconfig.osxkeychain @@ -0,0 +1,2 @@ +[credential] + helper = osxkeychain diff --git a/.github/macos-installer/assets/scripts/postinstall b/.github/macos-installer/assets/scripts/postinstall new file mode 100755 index 00000000000000..94056db9b7b864 --- /dev/null +++ b/.github/macos-installer/assets/scripts/postinstall @@ -0,0 +1,62 @@ +#!/bin/bash +INSTALL_DST="$2" +SCALAR_C_CMD="$INSTALL_DST/git/bin/scalar" +SCALAR_DOTNET_CMD="/usr/local/scalar/scalar" +SCALAR_UNINSTALL_SCRIPT="/usr/local/scalar/uninstall_scalar.sh" + +function cleanupScalar() +{ + echo "checking whether Scalar was installed" + if [ ! -f "$SCALAR_C_CMD" ]; then + echo "Scalar not installed; exiting..." + return 0 + fi + echo "Scalar is installed!" + + echo "looking for Scalar.NET" + if [ ! -f "$SCALAR_DOTNET_CMD" ]; then + echo "Scalar.NET not found; exiting..." + return 0 + fi + echo "Scalar.NET found!" + + currentUser=$(echo "show State:/Users/ConsoleUser" | scutil | awk '/Name :/ { print $3 }') + + # Re-register Scalar.NET repositories with the newly-installed Scalar + for repo in $($SCALAR_DOTNET_CMD list); do + ( + PATH="$INSTALL_DST/git/bin:$PATH" + sudo -u "$currentUser" scalar register $repo || \ + echo "warning: skipping re-registration of $repo" + ) + done + + # Uninstall Scalar.NET + echo "removing Scalar.NET" + + # Add /usr/local/bin to path - default install location of Homebrew + PATH="/usr/local/bin:$PATH" + if (sudo -u "$currentUser" brew list --cask scalar); then + # Remove from Homebrew + sudo -u "$currentUser" brew remove --cask scalar || echo "warning: Scalar.NET uninstall via Homebrew completed with code $?" + echo "Scalar.NET uninstalled via Homebrew!" + elif (sudo -u "$currentUser" brew list --cask scalar-azrepos); then + sudo -u "$currentUser" brew remove --cask scalar-azrepos || echo "warning: Scalar.NET with GVFS uninstall via Homebrew completed with code $?" + echo "Scalar.NET with GVFS uninstalled via Homebrew!" + elif [ -f $SCALAR_UNINSTALL_SCRIPT ]; then + # If not installed with Homebrew, manually remove package + sudo -S sh $SCALAR_UNINSTALL_SCRIPT || echo "warning: Scalar.NET uninstall completed with code $?" + echo "Scalar.NET uninstalled!" + else + echo "warning: Scalar.NET uninstall script not found" + fi + + # Re-create the Scalar symlink, in case it was removed by the Scalar.NET uninstall operation + mkdir -p $INSTALL_DST/bin + /bin/ln -Fs "$SCALAR_C_CMD" "$INSTALL_DST/bin/scalar" +} + +# Run Scalar cleanup (will exit if not applicable) +cleanupScalar + +exit 0 \ No newline at end of file diff --git a/.github/macos-installer/assets/uninstall.sh b/.github/macos-installer/assets/uninstall.sh new file mode 100755 index 00000000000000..4fc79fbaa2e652 --- /dev/null +++ b/.github/macos-installer/assets/uninstall.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e +if [ ! -r "/usr/local/git" ]; then + echo "Git doesn't appear to be installed via this installer. Aborting" + exit 1 +fi + +if [ "$1" != "--yes" ]; then + echo "This will uninstall git by removing /usr/local/git/, and symlinks" + printf "Type 'yes' if you are sure you wish to continue: " + read response +else + response="yes" +fi + +if [ "$response" == "yes" ]; then + # remove all of the symlinks we've created + pkgutil --files com.git.pkg | grep bin | while read f; do + if [ -L /usr/local/$f ]; then + sudo rm /usr/local/$f + fi + done + + # forget receipts. + pkgutil --packages | grep com.git.pkg | xargs -I {} sudo pkgutil --forget {} + echo "Uninstalled" + + # The guts all go here. + sudo rm -rf /usr/local/git/ +else + echo "Aborted" + exit 1 +fi + +exit 0 diff --git a/.github/macos-installer/entitlements.xml b/.github/macos-installer/entitlements.xml new file mode 100644 index 00000000000000..46f675661149b6 --- /dev/null +++ b/.github/macos-installer/entitlements.xml @@ -0,0 +1,12 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-library-validation + + + diff --git a/.github/release-homebrew.sh b/.github/release-homebrew.sh new file mode 100755 index 00000000000000..4a28c4a6bf73bb --- /dev/null +++ b/.github/release-homebrew.sh @@ -0,0 +1,159 @@ +#!/bin/sh +# +# Promote a microsoft/git release into the microsoft/homebrew-git tap. +# +# Usage: +# .github/release-homebrew.sh [--force] [] +# +# If TAG_NAME is omitted, the latest microsoft/git release is used. +# +# Downgrades require `--force`. +# +# Prerequisites: +# - `gh` authenticated (via `gh auth login`) as a user with push +# access to microsoft/homebrew-git. +# - `git`, `jq`, and `sed` on PATH. +# +# Given a release tag on microsoft/git (e.g. v2.54.0.vfs.0.4), this +# script looks up the macOS installer asset for that tag, extracts the +# SHA-256 digest reported by the GitHub Releases API (deliberately not +# re-hashed locally; see microsoft/homebrew-git#102), edits the +# `microsoft-git` cask in place preserving its indentation and quote +# style, pushes the update to the tap under a datetime-keyed branch, +# and opens a PR. +# +# This mirrors the behaviour of mjcheetham/update-homebrew@v1.5.1 +# invoked with `type: cask`, `alwaysUsePullRequest: true`. + +set -eu + +die () { + echo "error: $*" >&2 + exit 1 +} + +case "${1-}" in +--force) force=t; shift;; +*) force=;; +esac + +TAG_NAME=${1-} +if [ -z "$TAG_NAME" ]; then + echo "==> No tag given; resolving latest microsoft/git release" + TAG_NAME=$(gh release view -R microsoft/git \ + --json tagName --jq .tagName) + test -n "$TAG_NAME" || die "could not determine latest release tag" +fi + +echo "==> Tag: $TAG_NAME" + +version=${TAG_NAME#v} +echo "==> Version: $version" + +# Refuse to downgrade the cask and short-circuit no-op runs. Look up +# the version currently in the tap and compare before fetching the +# release JSON or cloning. +current_content=$(gh api \ + repos/microsoft/homebrew-git/contents/Casks/microsoft-git.rb \ + -H "Accept: application/vnd.github.raw") +current_version=$(printf '%s\n' "$current_content" | sed -nE \ + "s/^[[:space:]]*version *['\"]([^'\"]*)['\"].*/\\1/p") +test -n "$current_version" || die "could not parse current cask version" +echo "==> Current: $current_version" + +if [ "$version" = "$current_version" ]; then + echo "warning: cask is already at $version; nothing to do." >&2 + exit 0 +fi +lowest=$(printf '%s\n%s\n' "$version" "$current_version" | + sort -V | sed 1q) +if [ "$lowest" = "$version" ]; then + test -n "$force" || + die "regression: cask is at $current_version," \ + "refusing to downgrade to $version" + echo "warning: **downgrading** from $current_version to $version" >&2 +fi + +echo "==> Fetching release metadata" +release_json=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "repos/microsoft/git/releases/tags/$TAG_NAME") + +asset_pattern='git-(.*)\.pkg' +asset_json=$(jq -n \ + --argjson release "$release_json" \ + --arg pat "$asset_pattern" ' + [ $release.assets[] | select(.name | test($pat)) ] as $matches + | if ($matches | length) == 0 then + error("no asset matches pattern \($pat)") + elif ($matches | length) > 1 then + error("multiple assets match pattern \($pat): " + + ([$matches[].name] | join(", "))) + else $matches[0] end') + +digest=$(jq -n -r --argjson a "$asset_json" '$a.digest // ""') +case "$digest" in +sha256:*) sha256=${digest#sha256:} ;; +"") die "asset has no 'digest' field" ;; +*) die "asset digest is not sha256: $digest" ;; +esac + +# Enforce 64 lowercase hex chars without spawning grep. +case "$sha256" in +*[!0-9a-f]*|"") + die "asset digest is not lowercase hex: $sha256" ;; +esac +test ${#sha256} -eq 64 || + die "asset digest is not 64 chars long: $sha256" + +echo "==> Asset: $(jq -n -r --argjson a "$asset_json" '$a.name')" +echo "==> SHA-256: $sha256" + +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT + +echo "==> Cloning microsoft/homebrew-git" +REPO=microsoft/homebrew-git +gh repo clone "$REPO" "$workdir/homebrew-git" -- \ + --depth=1 --quiet + +cd "$workdir/homebrew-git" + +# Preserve existing indentation and quote style, replacing only the +# value; matches mjcheetham/update-homebrew's setField regex. The +# `file.new && mv -f file.new file` idiom sidesteps the +# incompatible `sed -i` spellings between GNU and BSD sed. +f=Casks/microsoft-git.rb +# Capture opening quote as \2, match value up to the matching quote. +q='(['\''"])[^'\''"]+\2' +sed -E \ + -e "s/^([[:space:]]*)version +$q/\\1version \\2$version\\2/" \ + -e "s/^([[:space:]]*)sha256 +$q/\\1sha256 \\2$sha256\\2/" \ + <"$f" >"$f.new" && +mv -f "$f.new" "$f" + +if git diff --quiet -- Casks/microsoft-git.rb; then + echo "==> No changes needed; cask is already at $version." + exit 0 +fi + +git --no-pager diff -- Casks/microsoft-git.rb + +BRANCH=update-$(date +%Y-%m-%d-%H-%M-%S) +git switch -c $BRANCH +TITLE="microsoft-git: update to $version" +git commit -m "$TITLE" \ + -- Casks/microsoft-git.rb + +git push origin HEAD + +echo "==> Pushed: $(git log -1 --format='%h %s')" + +pr_url=$(gh pr create \ + --repo "$REPO" \ + --head "$BRANCH" \ + --title "$TITLE" \ + --body "See https://github.com/microsoft/git/releases/tag/$TAG_NAME") + +echo "==> Created: $pr_url" diff --git a/.github/release-vfsforgit.sh b/.github/release-vfsforgit.sh new file mode 100755 index 00000000000000..29ecf30480c2b3 --- /dev/null +++ b/.github/release-vfsforgit.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# +# Promote a microsoft/git release into the microsoft/VFSForGit repo. +# +# Usage: +# .github/release-vfsforgit.sh [--force] [] +# +# If TAG_NAME is omitted, the latest microsoft/git release is used. +# +# Downgrades require `--force`. +# +# Prerequisites: +# - `gh` authenticated (via `gh auth login`) as a user with push +# access to microsoft/VFSForGit. +# - `git` and `sed` on PATH. +# +# Given a release tag on microsoft/git (e.g. v2.54.0.vfs.0.4), this +# script opens a pull request against microsoft/VFSForGit that bumps +# the `GIT_VERSION` default in `.github/workflows/build.yaml` so that +# VFSForGit builds pick up the newly promoted release by default. + +set -eu + +die () { + echo "error: $*" >&2 + exit 1 +} + +case "${1-}" in +--force) force=t; shift;; +*) force=;; +esac + +TAG_NAME=${1-} +if [ -z "$TAG_NAME" ]; then + echo "==> No tag given; resolving latest microsoft/git release" + TAG_NAME=$(gh release view -R microsoft/git \ + --json tagName --jq .tagName) + test -n "$TAG_NAME" || die "could not determine latest release tag" +fi + +echo "==> Tag: $TAG_NAME" + +REPO=microsoft/VFSForGit +BRANCH="automation/gitrelease-$TAG_NAME" +FILE=.github/workflows/build.yaml +RELEASE_URL="https://github.com/microsoft/git/releases/tag/$TAG_NAME" + +# Refuse to downgrade and short-circuit no-op runs. Read the current +# GIT_VERSION default straight from build.yaml on the VFSForGit +# default branch before cloning anything. +current_content=$(gh api "repos/$REPO/contents/$FILE" \ + -H "Accept: application/vnd.github.raw") +current_tag=$(printf '%s\n' "$current_content" | sed -nE \ + "/GIT_VERSION/s/.*\\|\\| *'([^']*)' *\\}\\}.*/\\1/p") +test -n "$current_tag" || die "could not parse current GIT_VERSION" +echo "==> Current: $current_tag" + +if [ "$TAG_NAME" = "$current_tag" ]; then + echo "warning: GIT_VERSION is already $TAG_NAME; nothing to do." >&2 + exit 0 +fi +lowest=$(printf '%s\n%s\n' "$TAG_NAME" "$current_tag" | + sort -V | sed 1q) +if [ "$lowest" = "$TAG_NAME" ]; then + test -n "$force" || + die "regression: GIT_VERSION is $current_tag," \ + "refusing to downgrade to $TAG_NAME" + echo "warning: **downgrading** from $current_tag to $TAG_NAME" >&2 +fi + +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT + +echo "==> Sparse-cloning $REPO" +gh repo clone "$REPO" "$workdir/vfsforgit" -- \ + --filter=blob:none --no-checkout --depth=1 --quiet +cd "$workdir/vfsforgit" +git sparse-checkout set "$FILE" +git checkout -b "$BRANCH" --quiet + +echo "==> Bumping GIT_VERSION in $FILE" +sed "/GIT_VERSION/s/|| '[^']*' }}/|| '$TAG_NAME' }}/" \ + <"$FILE" >"$FILE.new" && +mv -f "$FILE.new" "$FILE" + +git --no-pager diff -- "$FILE" + +git commit -m "Update default Microsoft Git version to $TAG_NAME" \ + -- "$FILE" + +git push origin "$BRANCH" + +pr_body="Update the default Microsoft Git version used by VFS for Git +to the newly promoted [\`$TAG_NAME\`]($RELEASE_URL) release." + +pr_url=$(gh pr create \ + --repo "$REPO" \ + --head "$BRANCH" \ + --title "Update default Microsoft Git version to $TAG_NAME" \ + --body "$pr_body") + +echo "==> Created: $pr_url" diff --git a/.github/release-winget.sh b/.github/release-winget.sh new file mode 100755 index 00000000000000..18c3fff5fb28d5 --- /dev/null +++ b/.github/release-winget.sh @@ -0,0 +1,190 @@ +#!/bin/sh +# +# Promote a microsoft/git release into the microsoft/winget-pkgs repo. +# +# Usage: +# .github/release-winget.sh [--force] [] +# +# If TAG_NAME is omitted, the latest microsoft/git release is used. +# +# Downgrades require `--force`. +# +# Prerequisites: +# - Runs on Windows (the winget authoring tool wingetcreate.exe is +# Windows-only). Use Git for Windows' bash, an MSYS2 shell, WSL +# Bash, or an equivalent. +# - `gh` authenticated (via `gh auth login`) as a user with (a) push +# access to a personal fork of microsoft/winget-pkgs and (b) +# permission to open a pull request against microsoft/winget-pkgs. +# wingetcreate will create the fork on the fly if it does not +# already exist. +# - `curl` and `jq` on PATH. +# +# Given a release tag on microsoft/git (e.g. v2.54.0.vfs.0.4), the +# script downloads wingetcreate, converts the tag to winget's dotted +# numeric version format (v2.54.0.vfs.0.4 -> 2.54.0.0.4), fetches the +# four installer URLs (x64 machine, x64 user, arm64 machine, arm64 +# user) from the corresponding GitHub release, builds an updated +# manifest for the Microsoft.Git package, syncs the operator's fork +# of microsoft/winget-pkgs with upstream, and submits the manifest +# via a pull request against microsoft/winget-pkgs. + +set -eu + +die () { + echo "error: $*" >&2 + exit 1 +} + +case "$(uname -s)" in +MINGW*|MSYS*|CYGWIN*) ;; # okay +Linux) + # Could be WSL + test -f /proc/sys/fs/binfmt_misc/WSLInterop || + die "this script requires Windows" + ;; +*) + die "this script requires Git for Windows / MSYS:" \ + "wingetcreate is a Windows-only tool" + ;; +esac + +case "${1-}" in +--force) force=t; shift;; +*) force=;; +esac + +TAG_NAME=${1-} +if [ -z "$TAG_NAME" ]; then + echo "==> No tag given; resolving latest microsoft/git release" + TAG_NAME=$(gh release view -R microsoft/git \ + --json tagName --jq .tagName) + test -n "$TAG_NAME" || die "could not determine latest release tag" +fi + +echo "==> Tag: $TAG_NAME" + +# Elide the leading 'v' and the 'vfs.' segment: +# v2.54.0.vfs.0.4 -> 2.54.0.0.4 +version=$(printf '%s' "${TAG_NAME#v}" | sed 's/vfs\.//') +echo "==> Version: $version" + +workdir=$(mktemp -d) +origdir="$(pwd)" +success=0 +cleanup () { + cd "$origdir" + if [ "$success" = 1 ]; then + rm -rf "$workdir" + else + echo "==> Workdir retained for inspection: $workdir" >&2 + fi +} +trap cleanup EXIT + +cd "$workdir" +echo "==> Working in $workdir" + +echo "==> Downloading wingetcreate" +test -x wingetcreate.exe || { + curl -fsSL https://aka.ms/wingetcreate/latest -o wingetcreate.exe && + chmod +x wingetcreate.exe +} || die "Could not initialize wingetcreate.exe" + +# Refuse to downgrade the package and short-circuit no-op runs. Look up +# the version currently in the manifest and compare before trying to +# update. +info="$(./wingetcreate.exe show Microsoft.Git)" +current_version=${info##*PackageVersion: } +current_version=${current_version%%[!0-9.]*} +test -n "$current_version" || die "could not parse current package version" +echo "==> Current: $current_version" + +if [ "$version" = "$current_version" ]; then + echo "warning: package is already at $version; nothing to do." >&2 + exit 0 +fi +lowest=$(printf '%s\n%s\n' "$version" "$current_version" | + sort -V | sed 1q) +if [ "$lowest" = "$version" ]; then + test -n "$force" || + die "regression: package is at $current_version," \ + "refusing to downgrade to $version" + echo "warning: **downgrading** from $current_version to $version" >&2 +fi + +echo "==> Fetching release metadata" +release_json=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "repos/microsoft/git/releases/tags/$TAG_NAME") + +pick_asset () { + # $1: jq regex to match the asset name. + jq -n -r --argjson r "$release_json" --arg pat "$1" ' + [ $r.assets[] | select(.name | test($pat)) ] as $m + | if $m | length == 0 then + error("no asset matches pattern \($pat)") + elif $m | length > 1 then + error("multiple assets match \($pat)") + else $m[0] end' +} + +x64_asset=$(pick_asset '64-bit\.exe$') +arm64_asset=$(pick_asset 'arm64\.exe$') + +# wingetcreate downloads the installer to compute its hash. Use the +# public browser URL for anonymous access rather than the API URL. +x64_url=$(printf '%s' "$x64_asset" | jq -r .browser_download_url) +arm64_url=$(printf '%s' "$arm64_asset" | jq -r .browser_download_url) + +echo "==> x64 asset: $(printf '%s' "$x64_asset" | jq -r .name)" +echo "==> arm64: $(printf '%s' "$arm64_asset" | jq -r .name)" + +# wingetcreate reads its GitHub token from this env var; hand it the +# operator's own gh session token so no PAT needs to be stashed. +WINGET_CREATE_GITHUB_TOKEN=$(gh auth token) +export WINGET_CREATE_GITHUB_TOKEN + +echo "==> Building manifest for Microsoft.Git $version" +./wingetcreate.exe update Microsoft.Git \ + -v "$version" \ + -o . \ + -u "$x64_url|x64|machine" \ + "$x64_url|x64|user" \ + "$arm64_url|arm64|machine" \ + "$arm64_url|arm64|user" + +# wingetcreate submit pushes to the operator's personal fork of +# microsoft/winget-pkgs and opens a PR from there. A stale fork makes +# submit fail with "The forked repository could not be synced with +# the upstream commits"; sync it first. If no fork exists yet (404), +# wingetcreate will create a fresh one at submit time, so treat that +# as fine. +user=$(gh api user --jq .login) +echo "==> Syncing $user/winget-pkgs fork with upstream" +sync_err=$(mktemp) +if gh api --silent --method POST \ + "repos/$user/winget-pkgs/merge-upstream" \ + -f branch=master 2>"$sync_err"; then + echo "==> Fork sync: OK" +elif grep -q '404' "$sync_err"; then + echo "==> Fork sync: no fork; will create on submit" +else + cat "$sync_err" >&2 + rm -f "$sync_err" + die "fork sync failed" +fi +rm -f "$sync_err" + +manifest_dir="$PWD/manifests/m/Microsoft/Git/$version" +echo "==> Submitting $manifest_dir" +submit_out=$(./wingetcreate.exe submit "$manifest_dir") +echo "$submit_out" + +pr_url=$(printf '%s\n' "$submit_out" | + grep -oE 'https://github\.com/microsoft/winget-pkgs/pull/[^ ]+' || + true) +test -z "$pr_url" || echo "==> Created: $pr_url" + +success=1 diff --git a/.github/scripts/codesign.sh b/.github/scripts/codesign.sh new file mode 100755 index 00000000000000..076b29f93be45e --- /dev/null +++ b/.github/scripts/codesign.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +sign_directory () { + ( + cd "$1" + for f in * + do + macho=$(file --mime $f | grep mach) + # Runtime sign dylibs and Mach-O binaries + if [[ $f == *.dylib ]] || [ ! -z "$macho" ]; + then + echo "Runtime Signing $f" + codesign -s "$IDENTITY" $f --timestamp --force --options=runtime --entitlements $ENTITLEMENTS_FILE + elif [ -d "$f" ]; + then + echo "Signing files in subdirectory $f" + sign_directory "$f" + + else + echo "Signing $f" + codesign -s "$IDENTITY" $f --timestamp --force + fi + done + ) +} + +for i in "$@" +do +case "$i" in + --payload=*) + SIGN_DIR="${i#*=}" + shift # past argument=value + ;; + --identity=*) + IDENTITY="${i#*=}" + shift # past argument=value + ;; + --entitlements=*) + ENTITLEMENTS_FILE="${i#*=}" + shift # past argument=value + ;; + *) + die "unknown option '$i'" + ;; +esac +done + +if [ -z "$SIGN_DIR" ]; then + echo "error: missing directory argument" + exit 1 +elif [ -z "$IDENTITY" ]; then + echo "error: missing signing identity argument" + exit 1 +elif [ -z "$ENTITLEMENTS_FILE" ]; then + echo "error: missing entitlements file argument" + exit 1 +fi + +echo "======== INPUTS ========" +echo "Directory: $SIGN_DIR" +echo "Signing identity: $IDENTITY" +echo "Entitlements: $ENTITLEMENTS_FILE" +echo "======== END INPUTS ========" + +sign_directory "$SIGN_DIR" diff --git a/.github/scripts/notarize.sh b/.github/scripts/notarize.sh new file mode 100755 index 00000000000000..9315d688afbd49 --- /dev/null +++ b/.github/scripts/notarize.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +for i in "$@" +do +case "$i" in + --package=*) + PACKAGE="${i#*=}" + shift # past argument=value + ;; + --keychain-profile=*) + KEYCHAIN_PROFILE="${i#*=}" + shift # past argument=value + ;; + *) + die "unknown option '$i'" + ;; +esac +done + +if [ -z "$PACKAGE" ]; then + echo "error: missing package argument" + exit 1 +elif [ -z "$KEYCHAIN_PROFILE" ]; then + echo "error: missing keychain profile argument" + exit 1 +fi + +# Exit as soon as any line fails +set -e + +# Send the notarization request +xcrun notarytool submit -v "$PACKAGE" -p "$KEYCHAIN_PROFILE" --wait + +# Staple the notarization ticket (to allow offline installation) +xcrun stapler staple -v "$PACKAGE" diff --git a/.github/scripts/symlink-git-hardlinks.rb b/.github/scripts/symlink-git-hardlinks.rb new file mode 100644 index 00000000000000..174802ccc85d93 --- /dev/null +++ b/.github/scripts/symlink-git-hardlinks.rb @@ -0,0 +1,19 @@ +#!/usr/bin/env ruby + +install_prefix = ARGV[0] +puts install_prefix +git_binary = File.join(install_prefix, '/usr/local/git/bin/git') + +[ + ['git' , File.join(install_prefix, '/usr/local/git/bin')], + ['../../bin/git', File.join(install_prefix, '/usr/local/git/libexec/git-core')] +].each do |link, path| + Dir.glob(File.join(path, '*')).each do |file| + next if file == git_binary + puts "#{file} #{File.size(file)} == #{File.size(git_binary)}" + next unless File.size(file) == File.size(git_binary) + puts "Symlinking #{file}" + puts `ln -sf #{link} #{file}` + exit $?.exitstatus if $?.exitstatus != 0 + end +end \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000000..1ab0b2d68f6a98 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,67 @@ +name: "CodeQL" + +on: + push: + pull_request: + workflow_dispatch: + schedule: + - cron: '0 3 * * 1' # Every Monday at 03:00 UTC + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ["cpp", "javascript"] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install dependencies + run: ci/install-dependencies.sh + if: matrix.language == 'cpp' + env: + jobname: codeql + CI_JOB_IMAGE: ubuntu-latest + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + config-file: ./.github/codeql/codeql-config.yml + + - name: Build + if: matrix.language == 'cpp' + run: | + cat /proc/cpuinfo + make -j$(nproc) + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + upload: False + output: sarif-results + + - name: debug + shell: bash + run: ls -la sarif-results + + - name: publish sarif for debugging + uses: actions/upload-artifact@v7 + with: + name: sarif-results-${{ matrix.language }} + path: sarif-results + + - name: Upload SARIF + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: sarif-results/${{ matrix.language }}.sarif diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71cabd013fb4f1..2c02644ebeda77 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,7 +190,7 @@ jobs: vs-build: name: win+VS build needs: ci-config - if: github.event.repository.owner.login == 'git-for-windows' && needs.ci-config.outputs.enabled == 'yes' + if: github.event.repository.owner.login == 'microsoft' && needs.ci-config.outputs.enabled == 'yes' env: NO_PERL: 1 GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" @@ -225,6 +225,9 @@ jobs: cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \ -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows -DHOST_CPU=${{ matrix.arch }} - name: MSBuild + env: + # Avoid vcpkg locking its executable while preparing telemetry. + VCPKG_DISABLE_METRICS: 1 run: | $sln = if (Test-Path git.slnx) { 'git.slnx' } else { 'git.sln' } msbuild $sln -property:Configuration=Release -property:Platform=${{ matrix.arch }} -maxCpuCount:4 @@ -479,7 +482,9 @@ jobs: - run: chmod a+w $GITHUB_ENV && sudo --preserve-env --set-home --user=builder ci/run-build-and-tests.sh - name: print test failures if: failure() && env.FAILED_TEST_ARTIFACTS != '' - run: sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh + run: | + chmod a+w "$GITHUB_ENV" && + sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v7 diff --git a/.github/workflows/monitor-components.yml b/.github/workflows/monitor-components.yml deleted file mode 100644 index e9f16e58fff87c..00000000000000 --- a/.github/workflows/monitor-components.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Monitor component updates - -# Git for Windows is a slightly modified subset of MSYS2. Some of its -# components are maintained by Git for Windows, others by MSYS2. To help -# keeping the former up to date, this workflow monitors the Atom/RSS feeds -# and opens new tickets for each new component version. - -on: - schedule: - - cron: "23 8,11,14,17 * * *" - workflow_dispatch: - -env: - CHARACTER_LIMIT: 5000 - MAX_AGE: 7d - -jobs: - job: - # Only run this in Git for Windows' fork - if: github.event.repository.owner.login == 'git-for-windows' - runs-on: ubuntu-latest - permissions: - issues: write - strategy: - matrix: - component: - - label: git - feed: https://github.com/git/git/tags.atom - - label: git-lfs - feed: https://github.com/git-lfs/git-lfs/tags.atom - - label: git-credential-manager - feed: https://github.com/git-ecosystem/git-credential-manager/tags.atom - - label: tig - feed: https://github.com/jonas/tig/tags.atom - - label: cygwin - feed: https://github.com/cygwin/cygwin/releases.atom - title-pattern: ^(?!.*newlib) - - label: msys2-runtime-package - feed: https://github.com/msys2/MSYS2-packages/commits/master/msys2-runtime.atom - - label: msys2-runtime - feed: https://github.com/msys2/msys2-runtime/commits/HEAD.atom - aggregate: true - - label: openssh - feed: https://github.com/openssh/openssh-portable/tags.atom - - label: libfido2 - feed: https://github.com/Yubico/libfido2/tags.atom - - label: libcbor - feed: https://github.com/PJK/libcbor/tags.atom - - label: openssl - feed: https://github.com/openssl/openssl/tags.atom - title-pattern: ^(?!.*alpha) - - label: gnutls - feed: https://gnutls.org/news.atom - - label: heimdal - feed: https://github.com/heimdal/heimdal/tags.atom - - label: git-sizer - feed: https://github.com/github/git-sizer/tags.atom - - label: gitflow - feed: https://github.com/petervanderdoes/gitflow-avh/tags.atom - - label: curl - feed: https://github.com/curl/curl/tags.atom - title-pattern: ^(?!rc-) - - label: mintty - feed: https://github.com/mintty/mintty/releases.atom - - label: 7-zip - feed: https://sourceforge.net/projects/sevenzip/rss?path=/7-Zip - aggregate: true - - label: bash - feed: https://git.savannah.gnu.org/cgit/bash.git/atom/?h=master - aggregate: true - - label: perl - feed: https://github.com/Perl/perl5/tags.atom - title-pattern: ^(?!.*(5\.[0-9]+[13579]|RC)) - - label: pcre2 - feed: https://github.com/PCRE2Project/pcre2/tags.atom - - label: mingw-w64-llvm - feed: https://github.com/msys2/MINGW-packages/commits/master/mingw-w64-llvm.atom - - label: innosetup - feed: https://github.com/jrsoftware/issrc/tags.atom - fail-fast: false - steps: - - uses: git-for-windows/rss-to-issues@v0 - with: - feed: ${{matrix.component.feed}} - prefix: "[New ${{matrix.component.label}} version]" - labels: component-update - github-token: ${{ secrets.GITHUB_TOKEN }} - character-limit: ${{ env.CHARACTER_LIMIT }} - max-age: ${{ env.MAX_AGE }} - aggregate: ${{matrix.component.aggregate}} - title-pattern: ${{matrix.component.title-pattern}} diff --git a/.github/workflows/nano-server.yml b/.github/workflows/nano-server.yml index 2e6da3ceea6f90..60e056a44bf8cc 100644 --- a/.github/workflows/nano-server.yml +++ b/.github/workflows/nano-server.yml @@ -28,11 +28,11 @@ jobs: docker run \ --user "ContainerAdministrator" \ -v "$WINDBG_DIR:C:/dbg" \ - -v "$(cygpath -aw /mingw64/bin):C:/mingw64-bin" \ + -v "$(cygpath -aw "/${MSYSTEM,,}/bin"):C:/mingw-bin" \ -v "$(cygpath -aw .):C:/test" \ $IMAGE pwsh.exe -Command ' - # Extend the PATH to include the `.dll` files in /mingw64/bin/ - $env:PATH += ";C:\mingw64-bin" + # Add runtime DLLs from the active SDK to PATH. + $env:PATH += ";C:\mingw-bin" # For each executable to test pick some no-operation set of # flags/subcommands or something that should quickly result in an diff --git a/.github/workflows/scalar-functional-tests.yml b/.github/workflows/scalar-functional-tests.yml new file mode 100644 index 00000000000000..672c505877b115 --- /dev/null +++ b/.github/workflows/scalar-functional-tests.yml @@ -0,0 +1,249 @@ +name: Scalar Functional Tests + +env: + SCALAR_REPOSITORY: microsoft/scalar + SCALAR_REF: main + DEBUG_WITH_TMATE: false + SCALAR_TEST_SKIP_VSTS_INFO: true + +on: + push: + branches: [ vfs-*, tentative/vfs-* ] + pull_request: + branches: [ vfs-*, features/* ] + +jobs: + scalar: + name: "Scalar Functional Tests" + + strategy: + fail-fast: false + matrix: + # Order by runtime (in descending order) + os: [windows-2022, macos-15, ubuntu-22.04] + # Scalar.NET used to be tested using `features: [false, experimental]` + # But currently, Scalar/C ignores `feature.scalar` altogether, so let's + # save some electrons and run only one of them... + features: [ignored] + exclude: + # The built-in FSMonitor is not (yet) supported on Linux + - os: ubuntu-22.04 + features: experimental + runs-on: ${{ matrix.os }} + + env: + BUILD_FRAGMENT: bin/Release/netcoreapp3.1 + GIT_FORCE_UNTRACKED_CACHE: 1 + + steps: + - name: Check out Git's source code + uses: actions/checkout@v6 + + - name: Setup build tools on Windows + if: runner.os == 'Windows' + uses: git-for-windows/setup-git-for-windows-sdk@v2 + + - name: Update the Git wrapper for UCRT64 + if: runner.os == 'Windows' + shell: pwsh + run: | + # Only needed for the MINGW64 to UCRT64 transition, until + # Actions runners ship Git for Windows v2.56.0 final. + if ($env:MSYSTEM -eq 'UCRT64' -and + !(Test-Path 'C:\Program Files\Git\ucrt64')) { + $url = 'https://github.com/git-for-windows/git/releases' + $url += '/download/v2.56.0-rc0.windows.1' + $zip = Join-Path $env:RUNNER_TEMP 'MinGit.zip' + Invoke-WebRequest -Uri "$url/MinGit-2.56.0-rc0-64-bit.zip" ` + -OutFile $zip + & "$env:SystemRoot\System32\tar.exe" -xf $zip ` + -C 'C:\Program Files\Git' cmd/git.exe cmd/scalar.exe + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + + - name: Provide a minimal `install` on Windows + if: runner.os == 'Windows' + shell: bash + run: | + test -x /usr/bin/install || + tr % '\t' >/usr/bin/install <<-\EOF + #!/bin/sh + + cmd=cp + while test $# != 0 + do + %case "$1" in + %-d) cmd="mkdir -p";; + %-m) shift;; # ignore mode + %*) break;; + %esac + %shift + done + + exec $cmd "$@" + EOF + + - name: Install build dependencies for Git (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext cargo + + - name: Build and install Git + shell: bash + env: + NO_TCLTK: Yup + run: | + # We do require a VFS version + def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)" + test -n "$def_ver" + + # Ensure that `git version` reflects DEF_VER + case "$(git describe --match "v[0-9]*vfs*" HEAD)" in + ${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this + *) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);; + esac + + SUDO= + extra= + case "${{ runner.os }}" in + Windows) + extra=DESTDIR=/c/Progra~1/Git + cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH + ;; + Linux) + SUDO=sudo + extra=prefix=/usr + ;; + macOS) + SUDO=sudo + extra=prefix=/opt/homebrew + ;; + esac + + $SUDO make -j5 $extra install + + - name: Ensure that we use the built Git and Scalar + shell: bash + run: | + type -p git + git version + case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac + type -p scalar + scalar version + case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac + + - name: Check out Scalar's source code + uses: actions/checkout@v6 + with: + fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works. + path: scalar + repository: ${{ env.SCALAR_REPOSITORY }} + ref: ${{ env.SCALAR_REF }} + + - name: Target .NET 9 + shell: bash + run: + csproj=scalar/Scalar.FunctionalTests/Scalar.FunctionalTests.csproj && + sed 's/netcoreapp3\.1/net9.0/g' <$csproj >$csproj.new && + mv $csproj.new $csproj && + + echo "BUILD_FRAGMENT=bin/Release/net9.0" >>$GITHUB_ENV && + + props=scalar/Directory.Build.props && + sed 's/\(\)[^<]*/\1osx-arm64/' <$props >$props.new && + mv $props.new $props + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: '9.0.306' + + - name: Install dependencies + run: dotnet restore + working-directory: scalar + env: + DOTNET_NOLOGO: 1 + + - name: Build + working-directory: scalar + run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS. + + - name: Setup platform (Linux) + if: runner.os == 'Linux' + run: | + echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV + + - name: Setup platform (Mac) + if: runner.os == 'macOS' + run: | + echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV + + - name: Setup platform (Windows) + if: runner.os == 'Windows' + run: | + echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV + echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV + + - name: Configure feature.scalar + run: git config --global feature.scalar ${{ matrix.features }} + + - id: functional_test + name: Functional test + timeout-minutes: 60 + working-directory: scalar + shell: bash + run: | + export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event" + export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf" + export GIT_TRACE2_EVENT_BRIEF=true + export GIT_TRACE2_PERF_BRIEF=true + mkdir -p "$TRACE2_BASENAME" + mkdir -p "$TRACE2_BASENAME/Event" + mkdir -p "$TRACE2_BASENAME/Perf" + git version --build-options + cd ../out + Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite + + - name: Force-stop FSMonitor daemons and Git processes (Windows) + if: runner.os == 'Windows' && (success() || failure()) + shell: bash + run: | + set -x + wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID + wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete + wmic process where "ExecutablePath Like '%git.exe'" delete + + - id: trace2_zip_unix + if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' ) + name: Zip Trace2 Logs (Unix) + shell: bash + working-directory: scalar + run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/ + + - id: trace2_zip_windows + if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' ) + name: Zip Trace2 Logs (Windows) + working-directory: scalar + run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }} + + - name: Archive Trace2 Logs + if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' ) + uses: actions/upload-artifact@v7 + with: + name: ${{ env.TRACE2_BASENAME }}.zip + path: scalar/${{ env.TRACE2_BASENAME }}.zip + retention-days: 3 + + # The GitHub Action `action-tmate` allows developers to connect to the running agent + # using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2 + # environment in `C:\msys64`, therefore it can be slightly tricky to interact with + # Git for Windows, which runs a slightly incompatible MSYS2 runtime). + - name: action-tmate + if: env.DEBUG_WITH_TMATE == 'true' && failure() + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true diff --git a/.github/workflows/vfs-functional-tests.yml b/.github/workflows/vfs-functional-tests.yml new file mode 100644 index 00000000000000..7c927ff3e4b6fe --- /dev/null +++ b/.github/workflows/vfs-functional-tests.yml @@ -0,0 +1,216 @@ +name: VFS for Git Functional Tests + +on: + push: + branches: [ vfs-*, tentative/vfs-* ] + pull_request: + branches: [ vfs-*, features/* ] + +permissions: + actions: read + contents: read + +jobs: + build: + runs-on: ${{ matrix.architecture == 'aarch64' && 'windows-11-arm' || 'windows-2022' }} + name: Build Git (${{ matrix.architecture }}) + + strategy: + matrix: + architecture: [ x86_64, aarch64 ] + fail-fast: false + + steps: + - name: Check out Git's source code + uses: actions/checkout@v6 + + - name: Setup build tools + uses: git-for-windows/setup-git-for-windows-sdk@v2 + with: + architecture: ${{ matrix.architecture }} + + - name: Provide a minimal `install` + shell: bash + run: | + test -x /usr/bin/install || + tr % '\t' >/usr/bin/install <<-\EOF + #!/bin/sh + + cmd=cp + while test $# != 0 + do + %case "$1" in + %-d) cmd="mkdir -p";; + %-m) shift;; # ignore mode + %*) break;; + %esac + %shift + done + + exec $cmd "$@" + EOF + + - name: Install GCC-compatible Rust target + shell: bash + run: | + # The hosted Windows runners ship a rustup-managed Rust whose + # default toolchain targets the MSVC ABI. That produces a + # `gitcore.lib` which the MinGW GCC used by the rest of the + # build cannot link. Install the precompiled `std` for a + # GCC-compatible target triple matching the MSYS2 subsystem; + # the Makefile selects the same triple via $(MSYSTEM) and + # passes it to `cargo build --target`. + case "$MSYSTEM" in + CLANGARM64) target=aarch64-pc-windows-gnullvm ;; + CLANG64) target=x86_64-pc-windows-gnullvm ;; + CLANG32) target=i686-pc-windows-gnullvm ;; + UCRT64) target=x86_64-pc-windows-gnullvm ;; + MINGW64) target=x86_64-pc-windows-gnu ;; + MINGW32) target=i686-pc-windows-gnu ;; + *) echo "::error::Unsupported MSYSTEM: $MSYSTEM"; exit 1 ;; + esac + rustup target add "$target" + + - name: Build and install Git + shell: bash + env: + NO_TCLTK: Yup + run: | + # We do require a VFS version + def_ver="$(sed -n '/^DEF_VER=/{ + s/^DEF_VER=\(.*vfs.*\)/\1/p + tq # already found a *.vfs.* one, skip next line + s/^DEF_VER=\(.*\)/\1.vfs.0.0/p + :q + q + }' GIT-VERSION-GEN)" + test -n "$def_ver" + + # VFSforGit cannot handle -rc versions; strip the `-rc` part, if any + case "$def_ver" in + *-rc*) def_ver=${def_ver%%-rc*}.vfs.${def_ver#*.vfs.};; + esac + + # Ensure that `git version` reflects DEF_VER + case "$(git describe --match "v[0-9]*vfs*" HEAD)" in + ${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this + *) echo ${def_ver}.NNN.g$(git rev-parse --short HEAD) >version;; + esac + + make -j5 DESTDIR="$GITHUB_WORKSPACE/MicrosoftGit/payload/${{ matrix.architecture }}" install + + # The test runners still ship the MINGW64 runtime. + if test "$MSYSTEM" = UCRT64 + then + cp /ucrt64/bin/*.dll \ + "$GITHUB_WORKSPACE/MicrosoftGit/payload/x86_64/ucrt64/bin/" + fi + + - name: Upload Git artifact + uses: actions/upload-artifact@v7 + with: + name: MicrosoftGit-${{ matrix.architecture }} + path: MicrosoftGit + + package: + runs-on: windows-2022 + name: Package Git + needs: build + + outputs: + vfs_run_id: ${{ steps.find_run.outputs.run_id }} + + steps: + - name: Download x86_64 build + uses: actions/download-artifact@v8 + with: + name: MicrosoftGit-x86_64 + path: MicrosoftGit + + - name: Download aarch64 build + uses: actions/download-artifact@v8 + with: + name: MicrosoftGit-aarch64 + path: MicrosoftGit + + - name: Create install script + shell: bash + run: | + cat >"$GITHUB_WORKSPACE/MicrosoftGit/install.bat" <<'BATCH' + @echo off + if "%PROCESSOR_ARCHITECTURE%"=="ARM64" ( + set GIT_PAYLOAD=%~dp0payload\aarch64 + ) else ( + set GIT_PAYLOAD=%~dp0payload\x86_64 + ) + rem Only needed for the MINGW64 to UCRT64 transition, until + rem Actions runners ship Git for Windows v2.56.0 final. + if not exist "%GIT_PAYLOAD%\ucrt64\bin\git.exe" goto install_git + if exist "C:\Program Files\Git\ucrt64" goto install_git + set "MINGIT_URL=https://github.com/git-for-windows/git/releases" + set "MINGIT_URL=%MINGIT_URL%/download/v2.56.0-rc0.windows.1" + "%SYSTEMROOT%\System32\curl.exe" -fL ^ + "%MINGIT_URL%/MinGit-2.56.0-rc0-64-bit.zip" ^ + -o "%RUNNER_TEMP%\MinGit.zip" || exit /b 1 + "%SYSTEMROOT%\System32\tar.exe" -xf "%RUNNER_TEMP%\MinGit.zip" ^ + -C "C:\Program Files\Git" cmd/git.exe || exit /b 1 + :install_git + echo Installing Git from %GIT_PAYLOAD% to "C:\Program Files\Git"... + robocopy "%GIT_PAYLOAD%" "C:\Program Files\Git" /E /NFL /NDL /NJH /NJS /NS /NC + if %ERRORLEVEL% GEQ 8 exit /b %ERRORLEVEL% + echo C:\Program Files\Git\cmd>>"%GITHUB_PATH%" + exit /b 0 + BATCH + + - name: Upload Git artifact + uses: actions/upload-artifact@v7 + with: + name: MicrosoftGit + path: MicrosoftGit + + - name: Find latest VFSForGit build + id: find_run + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + run_id=$(gh run list \ + --repo microsoft/VFSForGit \ + --workflow build.yaml \ + --branch master \ + --status success \ + --limit 1 \ + --json databaseId \ + -q '.[0].databaseId') + test -n "$run_id" || { echo "::error::No successful VFSForGit build found"; exit 1; } + + # If the run was skipped, follow the annotation to the real run. + # The skip notice may only appear on a re-attempt, so check + # attempts from latest to first. + attempts=$(gh api "repos/microsoft/VFSForGit/actions/runs/$run_id" \ + --jq '.run_attempt') + for attempt in $(seq "$attempts" -1 1); do + job_id=$(gh api "repos/microsoft/VFSForGit/actions/runs/$run_id/attempts/$attempt/jobs" \ + --jq '.jobs[] | select(.name == "Validation") | .id') + test -n "$job_id" || continue + skip_msg=$(gh api "repos/microsoft/VFSForGit/check-runs/$job_id/annotations" \ + --jq '[.[] | select(.message | startswith("Skipping:"))][0].message // empty') + test -n "$skip_msg" || continue + real_id=$(echo "$skip_msg" | sed -n 's|.*/runs/\([0-9]*\).*|\1|p') + if [ -n "$real_id" ] && [ "$real_id" != "$run_id" ]; then + echo "::notice::Skipped run $run_id (attempt $attempt), following to real run $real_id" + run_id=$real_id + fi + break + done + + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + echo ::notice::Using VFSForGit build with run_id=$run_id + + functional_tests: + name: Functional Tests + needs: package + uses: microsoft/VFSForGit/.github/workflows/functional-tests.yaml@master + with: + vfs_repository: microsoft/VFSForGit + vfs_run_id: ${{ needs.package.outputs.vfs_run_id }} diff --git a/.gitignore b/.gitignore index c730b1587e518a..ee534d07386c6c 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ /git-gc /git-get-tar-commit-id /git-grep +/git-gvfs-helper /git-hash-object /git-help /git-history @@ -180,6 +181,7 @@ /git-unpack-file /git-unpack-objects /git-update-index +/git-update-microsoft-git /git-update-ref /git-update-server-info /git-upload-archive @@ -263,3 +265,5 @@ Release/ CMakeSettings.json /contrib/libgit-rs/target /contrib/libgit-sys/target +/.github/codeql/.cache/ +/.github/codeql/codeql-pack.lock.yml diff --git a/AGENTS.md b/AGENTS.md index 7273095017731a..a387a425110291 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,8 +48,8 @@ to `PATH`, then invoke a non-login `bash -c` (replace `C:\git-sdk-64` with your SDK root): ```powershell -$env:MSYSTEM = "MINGW64" -$env:PATH = "C:\git-sdk-64\mingw64\bin;C:\git-sdk-64\usr\bin;" + $env:PATH +$env:MSYSTEM = "UCRT64" +$env:PATH = "C:\git-sdk-64\ucrt64\bin;C:\git-sdk-64\usr\bin;" + $env:PATH & C:\git-sdk-64\usr\bin\bash.exe -c "make -j15" ``` diff --git a/BRANCHES.md b/BRANCHES.md new file mode 100644 index 00000000000000..364158375e7d55 --- /dev/null +++ b/BRANCHES.md @@ -0,0 +1,59 @@ +Branches used in this repo +========================== + +The document explains the branching structure that we are using in the VFSForGit repository as well as the forking strategy that we have adopted for contributing. + +Repo Branches +------------- + +1. `vfs-#` + + These branches are used to track the specific version that match Git for Windows with the VFSForGit specific patches on top. When a new version of Git for Windows is released, the VFSForGit patches will be rebased on that windows version and a new gvfs-# branch created to create pull requests against. + + #### Examples + + ``` + vfs-2.27.0 + vfs-2.30.0 + ``` + + The versions of git for VFSForGit are based on the Git for Windows versions. v2.20.0.vfs.1 will correspond with the v2.20.0.windows.1 with the VFSForGit specific patches applied to the windows version. + +2. `vfs-#-exp` + + These branches are for releasing experimental features to early adopters. They + should contain everything within the corresponding `vfs-#` branch; if the base + branch updates, then merge into the `vfs-#-exp` branch as well. + +Tags +---- + +We are using annotated tags to build the version number for git. The build will look back through the commit history to find the first tag matching `v[0-9]*vfs*` and build the git version number using that tag. + +Full releases are of the form `v2.XX.Y.vfs.Z.W` where `v2.XX.Y` comes from the +upstream version and `Z.W` are custom updates within our fork. Specifically, +the `.Z` value represents the "compatibility level" with VFS for Git. Only +increase this version when making a breaking change with a released version +of VFS for Git. The `.W` version is used for minor updates between major +versions. + +Experimental releases are of the form `v2.XX.Y.vfs.Z.W.exp`. The `.exp` +suffix indicates that experimental features are available. The rest of the +version string comes from the full release tag. These versions will only +be made available as pre-releases on the releases page, never a full release. + +Forking +------- + +A personal fork of this repository and a branch in that repository should be used for development. + +These branches should be based on the latest vfs-# branch. If there are work in progress pull requests that you have based on a previous version branch when a new version branch is created, you will need to move your patches to the new branch to get them in that latest version. + +#### Example + +``` +git clone +git remote add ms https://github.com/Microsoft/git.git +git checkout -b my-changes ms/vfs-2.20.0 --no-track +git push -fu origin HEAD +``` diff --git a/Documentation/config.adoc b/Documentation/config.adoc index ec203ed7126c8b..2e9958f0d02730 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -504,6 +504,8 @@ include::config/gui.adoc[] include::config/guitool.adoc[] +include::config/gvfs.adoc[] + include::config/help.adoc[] include::config/hook.adoc[] @@ -546,6 +548,8 @@ include::config/pack.adoc[] include::config/pager.adoc[] +include::config/postcommand.adoc[] + include::config/pretty.adoc[] include::config/promisor.adoc[] diff --git a/Documentation/config/advice.adoc b/Documentation/config/advice.adoc index e182cc9838a4c0..1c3048f7e50742 100644 --- a/Documentation/config/advice.adoc +++ b/Documentation/config/advice.adoc @@ -178,7 +178,9 @@ all advice messages. Shown when the user tries to create a worktree from an invalid reference, to tell the user how to create a new unborn branch instead. - + useCoreConfigWriteLockTimeoutMSConfig:: + Advice shown if the deprecated 'core.configWriteLockTimeoutMS' + config setting is in use. useCoreFSMonitorConfig:: Advice shown if the deprecated 'core.useBuiltinFSMonitor' config setting is in use. diff --git a/Documentation/config/blame.adoc b/Documentation/config/blame.adoc index 4d047c17908cd6..23c64dee69e772 100644 --- a/Documentation/config/blame.adoc +++ b/Documentation/config/blame.adoc @@ -35,3 +35,21 @@ blame.markUnblamableLines:: blame.markIgnoredLines:: Mark lines that were changed by an ignored revision that we attributed to another commit with a '?' in the output of linkgit:git-blame[1]. + +blame.renames:: + If set to `false`, disable rename following in + linkgit:git-blame[1]. This option defaults to `true`. + +blame.renameThreshold:: + The minimum similarity threshold for rename detection in + linkgit:git-blame[1]; equivalent to the `-M` option of + linkgit:git-diff[1]. The value is a percentage (e.g. `50%`), + or a fraction between 0 and 1 (e.g. `0.5`). If not set, the + default is 50%. To limit blame to only follow exact renames, + set `blame.renameThreshold = 100%`. + +blame.renameLimit:: + The number of files to consider when performing rename + detection in linkgit:git-blame[1]; equivalent to the `-l` + option of linkgit:git-diff[1]. If not set, the default + value is currently 1000. diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 4d40c16ede9cfe..b9219080296e65 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -111,6 +111,14 @@ Version 2 uses an opaque string so that the monitor can return something that can be used to determine what files have changed without race conditions. +core.virtualFilesystem:: + If set, the value of this variable is used as a command which + will identify all files and directories that are present in + the working directory. Git will only track and update files + listed in the virtual file system. Using the virtual file system + will supersede the sparse-checkout settings which will be ignored. + See the "virtual file system" section of linkgit:githooks[5]. + core.trustctime:: If false, the ctime differences between the index and the working tree are ignored; useful when the inode change time @@ -781,6 +789,55 @@ core.multiPackIndex:: single index. See linkgit:git-multi-pack-index[1] for more information. Defaults to true. +core.gvfs:: + Enable the features needed for GVFS. This value can be set to true + to indicate all features should be turned on or the bit values listed + below can be used to turn on specific features. ++ +-- + GVFS_SKIP_SHA_ON_INDEX:: + Bit value 1 + Disables the calculation of the sha when writing the index + GVFS_MISSING_OK:: + Bit value 4 + Normally git write-tree ensures that the objects referenced by the + directory exist in the object database. This option disables this check. + GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT:: + Bit value 8 + When marking entries to remove from the index and the working + directory this option will take into account what the + skip-worktree bit was set to so that if the entry has the + skip-worktree bit set it will not be removed from the working + directory. This will allow virtualized working directories to + detect the change to HEAD and use the new commit tree to show + the files that are in the working directory. + GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK:: + Bit value 16 + While performing a fetch with a virtual file system we know + that there will be missing objects and we don't want to download + them just because of the reachability of the commits. We also + don't want to download a pack file with commits, trees, and blobs + since these will be downloaded on demand. This flag will skip the + checks on the reachability of objects during a fetch as well as + the upload pack so that extraneous objects don't get downloaded. + GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS:: + Bit value 64 + With a virtual file system we only know the file size before any + CRLF or smudge/clean filters processing is done on the client. + To prevent file corruption due to truncation or expansion with + garbage at the end, these filters must not run when the file + is first accessed and brought down to the client. Git.exe can't + currently tell the first access vs subsequent accesses so this + flag just blocks them from occurring at all. + GVFS_PREFETCH_DURING_FETCH:: + Bit value 128 + While performing a `git fetch` command, use the gvfs-helper to + perform a "prefetch" of commits and trees. +-- + +core.useGvfsHelper:: + TODO + core.sparseCheckout:: Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1] for more information. diff --git a/Documentation/config/diff.adoc b/Documentation/config/diff.adoc index 1135a62a0ad3de..c556470eba5a01 100644 --- a/Documentation/config/diff.adoc +++ b/Documentation/config/diff.adoc @@ -154,6 +154,13 @@ endif::git-diff[] `-l`. If not set, the default value is currently 1000. This setting has no effect if rename detection is turned off. +`diff.renameThreshold`:: + The minimum similarity threshold for rename detection; + equivalent to the `git diff` option `-M`. The value is a + percentage (e.g. `50%`), or a fraction between 0 and 1 + (e.g. `0.5`). If not set, the default is 50%. This setting + has no effect if rename detection is turned off. + `diff.renames`:: Whether and how Git detects renames. If set to `false`, rename detection is disabled. If set to `true`, basic rename diff --git a/Documentation/config/gvfs.adoc b/Documentation/config/gvfs.adoc new file mode 100644 index 00000000000000..7c1a569ce4612b --- /dev/null +++ b/Documentation/config/gvfs.adoc @@ -0,0 +1,68 @@ +gvfs.cache-server:: + When set, redirect all GVFS Protocol requests to this base URL instead + of the origin server. Individual verbs can be overridden with the + `gvfs..cache-server` config keys. + +gvfs..cache-server:: + Override the base value of `gvfs.cache-server` when using this specific + ``. The verbs available are: ++ +-- + prefetch:: + Use this cache server when prefetching commits and tree packfiles. + get:: + Use this cache server when downloading objects immediately via the + GET endpoint. + post:: + Use this cache server when downloading objects in batches using the + POST endpoint. +-- + +gvfs.sharedcache:: + When set, place all object data downloaded via the GVFS Protocol into + this Git alternate. + +gvfs.fallback:: + If set to `false`, then never fallback to the origin server when the cache + server fails to connect. This will alert users to failures with the cache + server, but avoid causing throttling on the origin server. + +gvfs.negativeRefCheck:: + When using the GVFS Protocol, the `core.gvfs` `GVFS_MISSING_OK` bit + allows objects behind local refs to be missing, so `git push` + normally feeds every advertised ref to `git pack-objects` as an + exclusion, even ones that are absent locally. `pack-objects` then + lazily downloads each missing exclusion while searching for delta + bases, issuing one object request per advertised ref. If set to + `true`, `git push` instead performs a non-fetching existence check + and omits any advertised object it does not have locally, matching + the behavior of Git without the `GVFS_MISSING_OK` bit. Defaults to + `false`. + +gvfs.sessionKey:: + If set to a string, then the value is a config key name that will be used + to set a prefix in the `X-Session-Id` header sent to the server for all + GVFS Protocol calls. This allows engineering systems to improve support + across client and server behavior, including any End User Pseodynomous + Identifiers (EUPI) that may be configured. The `X-Session-Id` will + include the SID of the current process in either case. + +gvfs.prefetchThreads:: + Set the number of parallel `index-pack` processes that run when + installing prefetch packfiles. The default value is `1`, which + processes packfiles sequentially without any thread infrastructure. + Setting this to a higher value (for example `4`) enables parallel + index-pack execution, which can significantly speed up the + installation of multiple prefetch packs. Values less than `1` are + treated as `1`. + +gvfs.postThreads:: + Set the number of parallel workers used when fetching objects + via HTTP POST requests. Each worker creates its own HTTP + connection and streams the response directly into an + `index-pack --stdin` child process. The default value is `1`, + which processes POST requests sequentially using the existing + code path. Setting this to a higher value (for example `4`) + downloads multiple batches of objects concurrently when the POST + block size is at least `100`. Values less than `1` are treated as + `1`. diff --git a/Documentation/config/index.adoc b/Documentation/config/index.adoc index 3eff42036033ea..0d6d05b70ce03d 100644 --- a/Documentation/config/index.adoc +++ b/Documentation/config/index.adoc @@ -1,3 +1,9 @@ +index.deleteSparseDirectories:: + When enabled, the cone mode sparse-checkout feature will delete + directories that are outside of the sparse-checkout cone, unless + such a directory contains an untracked, non-ignored file. Defaults + to true. + index.recordEndOfIndexEntries:: Specifies whether the index file should include an "End Of Index Entry" section. This reduces index load time on multiprocessor diff --git a/Documentation/config/merge.adoc b/Documentation/config/merge.adoc index 15a4c14c38aade..a525d96fa3947a 100644 --- a/Documentation/config/merge.adoc +++ b/Documentation/config/merge.adoc @@ -47,6 +47,13 @@ include::fmt-merge-msg.adoc[] currently defaults to 7000. This setting has no effect if rename detection is turned off. +`merge.renameThreshold`:: + The minimum similarity threshold for rename detection during + a merge. If not specified, defaults to the value of + `diff.renameThreshold`. See `diff.renameThreshold` for the + accepted value format. This setting has no effect if rename + detection is turned off. + `merge.renames`:: Whether Git detects renames. If set to `false`, rename detection is disabled. If set to `true`, basic rename detection is enabled. diff --git a/Documentation/config/postcommand.adoc b/Documentation/config/postcommand.adoc new file mode 100644 index 00000000000000..1ae7ce92f6ee22 --- /dev/null +++ b/Documentation/config/postcommand.adoc @@ -0,0 +1,13 @@ +postCommand.strategy:: + The `post-command` hook is run on every Git process by default. This + config option allows running the hook only conditionally, according + to these values: ++ +---- +`always`;; + run the `post-command` hook on every process (default). + +`worktree-change`;; + run the `post-command` hook only if the current process wrote to + the index and updated the worktree. +---- diff --git a/Documentation/config/protocol.adoc b/Documentation/config/protocol.adoc index a9bf187a933a24..01c399e99bb5d3 100644 --- a/Documentation/config/protocol.adoc +++ b/Documentation/config/protocol.adoc @@ -6,7 +6,7 @@ protocol.allow:: default policy of `never`, and all other protocols (including file) have a default policy of `user`. Supported policies: + --- +---- * `always` - protocol is always able to be used. @@ -18,7 +18,7 @@ protocol.allow:: execute clone/fetch/push commands without user input, e.g. recursive submodule initialization. --- +---- protocol..allow:: Set a policy to be used by protocol `` with clone/fetch/push @@ -26,7 +26,7 @@ protocol..allow:: + The protocol names currently used by git are: + --- +---- - `file`: any local file-based path (including `file://` URLs, or local paths) @@ -42,7 +42,7 @@ The protocol names currently used by git are: - any external helpers are named by their protocol (e.g., use `hg` to allow the `git-remote-hg` helper) --- +---- protocol.version:: If set, clients will attempt to communicate with a server @@ -51,7 +51,7 @@ protocol.version:: If unset, the default is `2`. Supported versions: + --- +---- * `0` - the original wire protocol. @@ -60,4 +60,4 @@ protocol.version:: * `2` - Wire protocol version 2, see linkgit:gitprotocol-v2[5]. --- +---- diff --git a/Documentation/config/push.adoc b/Documentation/config/push.adoc index 28132eedfee6c0..8fd64cfadf0073 100644 --- a/Documentation/config/push.adoc +++ b/Documentation/config/push.adoc @@ -17,7 +17,7 @@ (i.e. the fetch source is equal to the push destination), `upstream` is probably what you want. Possible values are: + --- +---- `nothing`;; do not push anything (error out) unless a refspec is given. This is primarily meant for people who want to @@ -70,7 +70,7 @@ branches outside your control. This used to be the default, but not since Git 2.0 (`simple` is the new default). --- +---- `push.followTags`:: If set to true, enable `--follow-tags` option by default. You diff --git a/Documentation/config/repo.adoc b/Documentation/config/repo.adoc index 7f8cd632965aab..5705280940f93f 100644 --- a/Documentation/config/repo.adoc +++ b/Documentation/config/repo.adoc @@ -3,9 +3,36 @@ repo.structure.*:: `git repo structure` command. + -- + nameRev:: + A boolean controlling whether revision names for ranked + objects are resolved and reported. Defaults to `true`. + The `--[no-]name-rev` option in linkgit:git-repo[1] + overrides this setting. top:: This integer value implies `--top=`, specifying the number of largest paths to report in each detail table. Must be non-negative; defaults to `0`, which disables the detail tables. + showBlobSizes:: + A non-negative integer specifying the default for + `--blob-sizes=` in linkgit:git-repo[1]. Defaults to `0`, + which disables the list of largest blobs. + showCommitParents:: + A non-negative integer specifying the default for + `--commit-parents=` in linkgit:git-repo[1]. Defaults to + `0`, which disables the list of commits with the most + parents. + showCommitSizes:: + A non-negative integer specifying the default for + `--commit-sizes=` in linkgit:git-repo[1]. Defaults to + `0`, which disables the list of largest commits. + showTreeEntries:: + A non-negative integer specifying the default for + `--tree-entries=` in linkgit:git-repo[1]. Defaults to + `0`, which disables the list of trees with the most + entries. + showTreeSizes:: + A non-negative integer specifying the default for + `--tree-sizes=` in linkgit:git-repo[1]. Defaults to `0`, + which disables the list of largest trees. -- diff --git a/Documentation/config/sendemail.adoc b/Documentation/config/sendemail.adoc index 5499f91036b3ec..4be6f1be22522a 100644 --- a/Documentation/config/sendemail.adoc +++ b/Documentation/config/sendemail.adoc @@ -74,7 +74,7 @@ the documentation of the email program of the same name. The differences and limitations from the standard formats are described below: + --- +---- `sendmail`;; * Quoted aliases and quoted addresses are not supported: lines that contain a `"` symbol are ignored. @@ -84,7 +84,7 @@ described below: * Warnings are printed on the standard error output for any explicitly unsupported constructs, and any other lines that are not recognized by the parser. --- +---- `sendemail.annotate`:: `sendemail.bcc`:: `sendemail.cc`:: diff --git a/Documentation/config/ssh.adoc b/Documentation/config/ssh.adoc index 2ca4bf93e1e30f..4f61c9351f3815 100644 --- a/Documentation/config/ssh.adoc +++ b/Documentation/config/ssh.adoc @@ -19,7 +19,7 @@ overridden via the environment variable `GIT_SSH_VARIANT`. The current command-line parameters used for each variant are as follows: + --- +---- * `ssh` - [-p port] [-4] [-6] [-o option] [username@]host command @@ -29,7 +29,7 @@ follows: * `tortoiseplink` - [-P port] [-4] [-6] -batch [username@]host command --- +---- + Except for the `simple` variant, command-line parameters are likely to change as git gains new features. diff --git a/Documentation/config/status.adoc b/Documentation/config/status.adoc index b5dd85b761f6f9..7911f4d24028cd 100644 --- a/Documentation/config/status.adoc +++ b/Documentation/config/status.adoc @@ -54,6 +54,12 @@ status.renameLimit:: in linkgit:git-status[1] and linkgit:git-commit[1]. Defaults to the value of diff.renameLimit. +status.renameThreshold:: + The minimum similarity threshold for rename detection in + linkgit:git-status[1] and linkgit:git-commit[1]. Defaults to + the value of diff.renameThreshold. See `diff.renameThreshold` + for the accepted value format. + status.renames:: Whether and how Git detects renames in linkgit:git-status[1] and linkgit:git-commit[1] . If set to "false", rename detection is @@ -75,11 +81,11 @@ status.showUntrackedFiles:: systems. So, this variable controls how the commands display the untracked files. Possible values are: + --- +---- * `no` - Show no untracked files. * `normal` - Show untracked files and directories. * `all` - Show also individual files in untracked directories. --- +---- + If this variable is not specified, it defaults to 'normal'. All usual spellings for Boolean value `true` are taken as `normal` @@ -102,3 +108,25 @@ status.submoduleSummary:: the --ignore-submodules=dirty command-line option or the 'git submodule summary' command, which shows a similar output but does not honor these settings. + +status.deserializePath:: + EXPERIMENTAL, Pathname to a file containing cached status results + generated by `--serialize`. This will be overridden by + `--deserialize=` on the command line. If the cache file is + invalid or stale, git will fall-back and compute status normally. + +status.deserializeWait:: + EXPERIMENTAL, Specifies what `git status --deserialize` should do + if the serialization cache file is stale and whether it should + fall-back and compute status normally. This will be overridden by + `--deserialize-wait=` on the command line. ++ +---- +* `fail` - cause git to exit with an error when the status cache file +is stale; this is intended for testing and debugging. +* `block` - cause git to spin and periodically retry the cache file +every 100 ms; this is intended to help coordinate with another git +instance concurrently computing the cache file. +* `no` - to immediately fall-back if cache file is stale. This is the default. +* `` - time (in tenths of a second) to spin and retry. +---- diff --git a/Documentation/config/survey.adoc b/Documentation/config/survey.adoc index 9e594a2092f225..0bee888d1b9d74 100644 --- a/Documentation/config/survey.adoc +++ b/Documentation/config/survey.adoc @@ -1,14 +1,3 @@ survey.*:: - These variables adjust the default behavior of the `git survey` - command. The intention is that this command could be run in the - background with these options. -+ --- - verbose:: - This boolean value implies the `--[no-]verbose` option. - progress:: - This boolean value implies the `--[no-]progress` option. - top:: - This integer value implies `--top=`, specifying the - number of entries in the detail tables. --- + Deprecated and ignored by `git survey`. See the "MIGRATION" section + of linkgit:git-survey[1] for replacements and changed defaults. diff --git a/Documentation/config/trace2.adoc b/Documentation/config/trace2.adoc index 05639ce33f908a..f5d730934c8354 100644 --- a/Documentation/config/trace2.adoc +++ b/Documentation/config/trace2.adoc @@ -16,7 +16,7 @@ trace2.eventTarget:: This variable controls the event target destination. It may be overridden by the `GIT_TRACE2_EVENT` environment variable. The following table shows possible values. -+ + include::../trace2-target-values.adoc[] trace2.normalBrief:: diff --git a/Documentation/git-blame.adoc b/Documentation/git-blame.adoc index 2b74e455997c8c..811433b41dd5ae 100644 --- a/Documentation/git-blame.adoc +++ b/Documentation/git-blame.adoc @@ -24,10 +24,11 @@ When specified one or more times, `-L` restricts annotation to the requested lines. The origin of lines is automatically followed across whole-file -renames (currently there is no option to turn the rename-following -off). To follow lines moved from one file to another, or to follow -lines that were copied and pasted from another file, etc., see the -`-C` and `-M` options. +renames. This can be disabled with `blame.renames`, and the minimum +similarity threshold can be adjusted with `blame.renameThreshold` +(see linkgit:git-config[1]). To follow lines moved from one file to +another, or to follow lines that were copied and pasted from another +file, etc., see the `-C` and `-M` options. The report does not tell you anything about lines which have been deleted or replaced; you need to use a tool such as `git diff` or the "pickaxe" diff --git a/Documentation/git-maintenance.adoc b/Documentation/git-maintenance.adoc index bda616f14c45d9..9dd4d15871a55c 100644 --- a/Documentation/git-maintenance.adoc +++ b/Documentation/git-maintenance.adoc @@ -70,6 +70,7 @@ task: * `prefetch`: hourly. * `loose-objects`: daily. * `incremental-repack`: daily. +* `cache-local-objects`: weekly. -- + `git maintenance register` will also disable foreground maintenance by @@ -185,6 +186,13 @@ worktree-prune:: The `worktree-prune` task deletes stale or broken worktrees. See linkgit:git-worktree[1] for more information. +cache-local-objects:: + The `cache-local-objects` task only operates on Scalar or VFS for Git + repositories (cloned with either `scalar clone` or `gvfs clone`) that + have the `gvfs.sharedCache` configuration setting present. This task + migrates pack files and loose objects from the repository's object + directory in to the shared volume cache. + OPTIONS ------- --auto:: diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index fd498abb7fa658..1c5c37be124cf8 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -61,15 +61,26 @@ supported: following kinds of information are reported: + * Reference counts categorized by type +* Symbolic reference counts +* Loose and packed reference counts (with the `files` reference backend) +* Maximum and total reference-name lengths for local and remote references * Reachable object counts categorized by type * Total inflated size of reachable objects by type * Total disk size of reachable objects by type * Largest reachable objects in the repository by type +* Histograms of commit, tree, and blob sizes, tree entry counts, and + commit parent counts * Optionally, the top-_N_ largest paths by count, on-disk size, and inflated size (see `--top` below) + By default every reference enumerated by `for-each-ref` contributes to the counts and object walk. Use `--ref-filter` to narrow the scope. ++ +Symbolic references also contribute to their usual reference-type counts. +Reference-name lengths are measured in bytes, including the full `refs/` +prefix. Remote references are those under `refs/remotes/`; all other +references, including tags, are considered local for these measurements. +Loose and packed counts are zero for reference backends other than `files`. `--ref-filter=`;; Only count references whose full name matches one of the @@ -83,6 +94,14 @@ the counts and object walk. Use `--ref-filter` to narrow the scope. `--ref-filter='refs/remotes/origin/*'` for a single remote's branches. + `--name-rev`;; + `--no-name-rev`;; + Resolve revision names for the commits associated with + ranked objects. Enabled by default. `--no-name-rev` skips + the lookup and omits `name_rev` from every output format. + The default can be set with `repo.structure.nameRev`; + explicit command-line options override this setting. + `--top=`;; Also report the _n_ largest paths in the repository, separately for trees and blobs and separately ranked by @@ -91,6 +110,33 @@ the counts and object walk. Use `--ref-filter` to narrow the scope. can also be set via the `repo.structure.top` configuration variable; an explicit `--top=` on the command line overrides the configured value. + + `--commit-parents=`;; + Report the _n_ commits with the most parents. + + `--commit-sizes=`;; + Report the _n_ largest commits by inflated size in bytes, + usually those with the largest commit messages. + + `--tree-entries=`;; + Report the _n_ trees with the most entries (files and + subdirectories). + + `--tree-sizes=`;; + Report the _n_ largest trees by inflated size in bytes. + This ranking can differ from the entry-count ranking + when entry names have very different lengths. + + `--blob-sizes=`;; + Report the _n_ largest blobs by inflated size in bytes. ++ +These five object-list limits are independent of each other and of `--top`, +which ranks paths rather than individual objects. Each limit must be +non-negative and defaults to `0`, which disables that list. Configuration +variables under `repo.structure.show*` provide defaults (see "CONFIGURATION" +below); an explicit command-line limit overrides the corresponding default. +Lists show the measured values and OIDs in descending order, with at most +_n_ entries, and honor `--ref-filter`. + The output format can be chosen through the flag `--format`. Three formats are supported: @@ -115,6 +161,58 @@ supported: + `-z` is an alias for `--format=nul`. +STRUCTURE HISTOGRAMS +-------------------- +Size histograms group commits, trees, and blobs by inflated size in bytes: +`0..15`, `16..255`, `256..4095`, and so on. Tree entry histograms use the +ranges `0..3`, `4..15`, `16..63`, and so on. Each bucket reports the number +of objects and their total inflated and on-disk sizes. Empty buckets are +omitted. + +The commit parent histogram reports exact parent counts from 0 through 30. +Its final bucket, shown as `31+` in tables, includes all commits with at +least 31 parents. + +In `lines` and `nul` output, size histogram keys have the form +`objects..histogram.size..`, where `` is +`commits`, `trees`, or `blobs`, and `` is `count`, `inflated_size`, +or `disk_size`. Tree entry histograms use +`objects.trees.histogram.entries..`. Bucket numbers start +at zero for the first range listed above. Parent counts use +`objects.commits.histogram.parents..count`, with `31` identifying +the final bucket. All histograms honor `--ref-filter`. + +STRUCTURE OBJECT LISTS +---------------------- +In `lines` and `nul` output, the optional object lists use keys of the form +`objects..largest...`. Ranks start at 1. +For commits, the dimensions are `by_parents` and `by_size`; for trees they +are `by_entries` and `by_size`; blobs support `by_size`. Each ranked object +has an `oid` field. Its value field is named `parents` for `by_parents`, +`entries` for `by_entries`, or `inflated_size` for `by_size`. Parent counts +in these lists are not capped by the histogram's final bucket. + +Tree and blob entries also have a `path` field containing the first path +associated with that object by the path walk. This is a suggested location, +not necessarily a path in `HEAD`: an unchanged object may occur at multiple +paths. An empty path denotes a root tree. Table and `lines` output quote +unusual paths according to `core.quotePath`; `nul` output preserves the +path bytes without quoting. + +Entries also have a `commit_oid` field when an associated commit is known. +Currently this is available only for commit objects, where it equals +`oid`. Table output shows this ID as `(commit )`. + +The `name_rev` field contains the result of passing `commit_oid` to +`git name-rev --name-only --annotate-stdin`. This uses all refs for naming, +even when `--ref-filter` limits the objects counted. An object ID is kept +unchanged if no name is found. Names appear in parentheses in tables and +use the same quoting rules as paths in all output formats. + +No name lookup is performed for entries without a `commit_oid`. If lookup +fails, a warning is printed and `name_rev` is omitted; the other statistics +are still reported. + INFO KEYS --------- In order to obtain a set of values from `git repo info`, you should provide @@ -167,6 +265,10 @@ using the `nul` format: git repo info --format=nul layout.bare layout.shallow ------------ +CONFIGURATION +------------- +include::config/repo.adoc[] + SEE ALSO -------- linkgit:git-rev-parse[1] diff --git a/Documentation/git-status.adoc b/Documentation/git-status.adoc index 9acca52bfb1abd..c24e2edcc7f76e 100644 --- a/Documentation/git-status.adoc +++ b/Documentation/git-status.adoc @@ -154,6 +154,21 @@ ignored, then the directory is not shown, but all contents are shown. threshold. See also linkgit:git-diff[1] `--find-renames`. +`--serialize[=]`:: + (EXPERIMENTAL) Serialize raw status results to a file or stdout + in a format suitable for use by `--deserialize`. If a path is + given, serialize data will be written to that path *and* normal + status output will be written to stdout. If path is omitted, + only binary serialization data will be written to stdout. + +`--deserialize[=]`:: + (EXPERIMENTAL) Deserialize raw status results from a file or + stdin rather than scanning the worktree. If `` is omitted + and `status.deserializePath` is unset, input is read from stdin. +`--no-deserialize`:: + (EXPERIMENTAL) Disable implicit deserialization of status results + from the value of `status.deserializePath`. + `...`:: See the 'pathspec' entry in linkgit:gitglossary[7]. @@ -432,6 +447,26 @@ quoted as explained for the configuration variable `core.quotePath` (see linkgit:git-config[1]). +SERIALIZATION and DESERIALIZATION (EXPERIMENTAL) +------------------------------------------------ + +The `--serialize` option allows git to cache the result of a +possibly time-consuming status scan to a binary file. A local +service/daemon watching file system events could use this to +periodically pre-compute a fresh status result. + +Interactive users could then use `--deserialize` to simply +(and immediately) print the last-known-good result without +waiting for the status scan. + +The binary serialization file format includes some worktree state +information allowing `--deserialize` to reject the cached data +and force a normal status scan if, for example, the commit, branch, +or status modes/options change. The format cannot, however, indicate +when the cached data is otherwise stale -- that coordination belongs +to the task driving the serializations. + + CONFIGURATION ------------- diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc index e97656b49dee75..4d5ca497a8e411 100644 --- a/Documentation/git-survey.adoc +++ b/Documentation/git-survey.adoc @@ -44,8 +44,8 @@ Ref Selection The following options control the set of refs that `git survey` will examine. By default, `git survey` will look at tags, local branches, and remote refs. -If any of the following options are given, the default set is cleared and -only refs for the given options are added. +If any ref-selection option other than `--detached` is given, the default +set is cleared and only refs for the given options are added. --all-refs:: Use all refs. This includes local branches, tags, remote refs, @@ -61,11 +61,44 @@ only refs for the given options are added. Add remote branches (`refs/remote/`) to the set. --detached:: - Add HEAD to the set. + Ignored with a warning. This option does not affect ref selection. --other:: Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set. +MIGRATION +--------- +`git repo structure` selects all refs by default. To retain the survey +default scope, pass `--ref-filter='refs/heads/*'`, +`--ref-filter='refs/tags/*'`, and `--ref-filter='refs/remotes/*'`. + +Neither command separately enumerates detached `HEAD`, even with +`git survey --all-refs`. To include history reachable only from `HEAD`, +first create a branch or tag pointing to `HEAD` and include that ref in +the selection. The new ref is also counted. + +For path detail tables, `git repo structure` defaults to `--top=0`; use +`--top=10` or `repo.structure.top=10` for the survey default. The shim +always supplies `--top` (10 unless overridden on its command line), so +`repo.structure.top` does not change `git survey`'s limit. + +`--commit-parents=`, `--commit-sizes=`, `--tree-entries=`, +`--tree-sizes=`, `--blob-sizes=`, and `--[no-]name-rev` retain their +names in `git repo structure`, but are not accepted by the shim. Each +object-list limit defaults to 0. + +Name lookup for ranked commits is now enabled by default. Use +`git repo structure --no-name-rev` or set `repo.structure.nameRev=false` +to opt out; the configuration also applies through `git survey`. + +All `survey.*` settings are ignored. For `top`, `nameRev`, +`showCommitParents`, `showCommitSizes`, `showTreeEntries`, `showTreeSizes`, +and `showBlobSizes`, replace the `survey.` prefix with `repo.structure.` +when invoking `git repo structure` directly. Use `--progress` or +`--no-progress` instead of `survey.progress`; no `repo.structure.progress` +exists. `survey.verbose` has no replacement; the shim ignores `--verbose` +with a warning. + OUTPUT ------ diff --git a/Documentation/git-update-microsoft-git.adoc b/Documentation/git-update-microsoft-git.adoc new file mode 100644 index 00000000000000..724bfc172f8ab7 --- /dev/null +++ b/Documentation/git-update-microsoft-git.adoc @@ -0,0 +1,24 @@ +git-update-microsoft-git(1) +=========================== + +NAME +---- +git-update-microsoft-git - Update the installed version of Git + + +SYNOPSIS +-------- +[verse] +'git update-microsoft-git' + +DESCRIPTION +----------- +This version of Git is based on the Microsoft fork of Git, which +has custom capabilities focused on supporting monorepos. This +command checks for the latest release of that fork and installs +it on your machine. + + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/githooks.adoc b/Documentation/githooks.adoc index ed045940d18ba5..29d63c7ed2c318 100644 --- a/Documentation/githooks.adoc +++ b/Documentation/githooks.adoc @@ -760,6 +760,26 @@ and "0" meaning they were not. Only one parameter should be set to "1" when the hook runs. The hook running passing "1", "1" should not be possible. +virtualFilesystem +~~~~~~~~~~~~~~~~~~ + +"Virtual File System" allows populating the working directory sparsely. +The projection data is typically automatically generated by an external +process. Git will limit what files it checks for changes as well as which +directories are checked for untracked files based on the path names given. +Git will also only update those files listed in the projection. + +The hook is invoked when the configuration option core.virtualFilesystem +is set. It takes one argument, a version (currently 1). + +The hook should output to stdout the list of all files in the working +directory that git should track. The paths are relative to the root +of the working directory and are separated by a single NUL. Full paths +('dir1/a.txt') as well as directories are supported (ie 'dir1/'). + +The exit status determines whether git will use the data from the +hook. On error, git will abort the command with an error message. + SEE ALSO -------- linkgit:git-hook[1] diff --git a/Documentation/lint-manpages.sh b/Documentation/lint-manpages.sh index a0ea572382d8d9..53c7ed9f12ec66 100755 --- a/Documentation/lint-manpages.sh +++ b/Documentation/lint-manpages.sh @@ -27,6 +27,8 @@ check_missing_docs () ( git-init-db) continue;; git-remote-*) continue;; git-stage) continue;; + git-gvfs-helper) continue;; + git-update-microsoft-git) continue;; git-legacy-*) continue;; git-?*--?* ) continue ;; esac diff --git a/Documentation/meson.build b/Documentation/meson.build index 605c0b5673205a..e74207b2aa2913 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -153,6 +153,7 @@ manpages = { 'git-unpack-file.adoc' : 1, 'git-unpack-objects.adoc' : 1, 'git-update-index.adoc' : 1, + 'git-update-microsoft-git.adoc' : 1, 'git-update-ref.adoc' : 1, 'git-update-server-info.adoc' : 1, 'git-upload-archive.adoc' : 1, diff --git a/Documentation/scalar.adoc b/Documentation/scalar.adoc index 5252fb134a47ab..7405ae8469bea6 100644 --- a/Documentation/scalar.adoc +++ b/Documentation/scalar.adoc @@ -9,7 +9,9 @@ SYNOPSIS -------- [verse] scalar clone [--single-branch] [--branch ] [--full-clone] - [--[no-]src] [--[no-]tags] [--[no-]maintenance] [] + [--[no-]src] [--[no-]tags] [--[no-]maintenance] [--[no-]prefetch] + [--cache-server-url ] [--[verb]-cache-server-url ] + [--local-cache-path ] [] scalar list scalar register [--[no-]maintenance] [] scalar unregister [] @@ -17,6 +19,7 @@ scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) scalar reconfigure [--maintenance=(enable|disable|keep)] [ --all | ] scalar diagnose [] scalar delete +scalar cache-server ( --get | --set | --list [] ) [] DESCRIPTION ----------- @@ -107,6 +110,59 @@ cloning. If the HEAD at the remote did not point at any branch when background maintenance feature. Use the `--no-maintenance` to skip this configuration. +--prefetch:: +--no-prefetch:: + By default, when cloning with the GVFS Protocol, `scalar clone` + issues a `/gvfs/prefetch` request to hydrate the local object cache + with the commits and trees that back the checked-out branch. Use + `--no-prefetch` to skip that request during the clone so that the + initial worktree becomes ready as quickly as possible. The tip commit + and the trees needed for the initial checkout are still downloaded + through the `/gvfs/objects` endpoint. ++ +This only affects the initial fetch performed by `scalar clone`: the +prefetched data is still downloaded by the next `git fetch` (including the +background maintenance `prefetch` task), so the object cache is populated +shortly afterwards. This option has no effect when the GVFS Protocol is not +in use. + +--local-cache-path :: + Override the path to the local cache root directory; Pre-fetched objects + are stored into a repository-dependent subdirectory of that path. ++ +The default is `:\.scalarCache` on Windows (on the same drive as the +clone), and `~/.scalarCache` on macOS. + +--cache-server-url :: + Retrieve missing objects from the specified remote, which is expected to + understand the GVFS protocol. + +--[verb]-cache-server-url :: + Set the appropriate `gvfs..cache-server` config value that overrides + the provided `--cache-server-url` or the dynamically discovered URL. The + list of allowed verbs is `prefetch`, `get`, and `post`. + +--gvfs-protocol:: +--no-gvfs-protocol:: + When cloning from a `` with either `dev.azure.com` or + `visualstudio.com` in the name, `scalar clone` will attempt to use the GVFS + Protocol to access Git objects, specifically from a cache server when + available, and will fail to clone if there is an error over that protocol. + + To enable the GVFS Protocol regardless of the origin ``, use + `--gvfs-protocol`. This will cause `scalar clone` to fail when the origin + server fails to provide a valid response to the `gvfs/config` endpoint. + + To disable the GVFS Protocol, use `--no-gvfs-protocol` and `scalar clone` + will only use the Git protocol, starting with a partial clone. This can be + helpful if your `` points to Azure Repos but the repository does not + have GVFS cache servers enabled. It is likely more efficient to use its + partial clone functionality through the Git protocol. + + Previous versions of `scalar clone` could fall back to a partial clone over + the Git protocol if there is any issue gathering GVFS configuration + information from the origin server. + List ~~~~ @@ -197,6 +253,27 @@ delete :: This subcommand lets you delete an existing Scalar enlistment from your local file system, unregistering the repository. +Cache-server +~~~~~~~~~~~~ + +cache-server ( --get | --set | --list [] ) []:: + This command lets you query or set the GVFS-enabled cache server used + to fetch missing objects. + +--get:: + This is the default command mode: query the currently-configured cache + server URL, if any. + +--list:: + Access the `gvfs/info` endpoint of the specified remote (default: + `origin`) to figure out which cache servers are available, if any. ++ +In contrast to the `--get` command mode (which only accesses the local +repository), this command mode triggers a request via the network that +potentially requires authentication. If authentication is required, the +configured credential helper is employed (see linkgit:git-credential[1] +for details). + RECOMMENDED CONFIG VALUES ------------------------- @@ -347,6 +424,9 @@ status.aheadBehind=false:: message that can be disabled by disabling the `advice.statusAheadBehind` config. +core.configLockTimeout:: + Sets a timeout to work gracefully around Git config write contention. + The following settings are different based on which platform is in use: core.untrackedCache=(true|false):: diff --git a/Documentation/technical/read-object-protocol.txt b/Documentation/technical/read-object-protocol.txt new file mode 100644 index 00000000000000..a893b46e7c28a9 --- /dev/null +++ b/Documentation/technical/read-object-protocol.txt @@ -0,0 +1,102 @@ +Read Object Process +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The read-object process enables Git to read all missing blobs with a +single process invocation for the entire life of a single Git command. +This is achieved by using a packet format (pkt-line, see technical/ +protocol-common.txt) based protocol over standard input and standard +output as follows. All packets, except for the "*CONTENT" packets and +the "0000" flush packet, are considered text and therefore are +terminated by a LF. + +Git starts the process when it encounters the first missing object that +needs to be retrieved. After the process is started, Git sends a welcome +message ("git-read-object-client"), a list of supported protocol version +numbers, and a flush packet. Git expects to read a welcome response +message ("git-read-object-server"), exactly one protocol version number +from the previously sent list, and a flush packet. All further +communication will be based on the selected version. + +The remaining protocol description below documents "version=1". Please +note that "version=42" in the example below does not exist and is only +there to illustrate how the protocol would look with more than one +version. + +After the version negotiation Git sends a list of all capabilities that +it supports and a flush packet. Git expects to read a list of desired +capabilities, which must be a subset of the supported capabilities list, +and a flush packet as response: +------------------------ +packet: git> git-read-object-client +packet: git> version=1 +packet: git> version=42 +packet: git> 0000 +packet: git< git-read-object-server +packet: git< version=1 +packet: git< 0000 +packet: git> capability=get +packet: git> capability=have +packet: git> capability=put +packet: git> capability=not-yet-invented +packet: git> 0000 +packet: git< capability=get +packet: git< 0000 +------------------------ +The only supported capability in version 1 is "get". + +Afterwards Git sends a list of "key=value" pairs terminated with a flush +packet. The list will contain at least the command (based on the +supported capabilities) and the sha1 of the object to retrieve. Please +note, that the process must not send any response before it received the +final flush packet. + +When the process receives the "get" command, it should make the requested +object available in the git object store and then return success. Git will +then check the object store again and this time find it and proceed. +------------------------ +packet: git> command=get +packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05 +packet: git> 0000 +------------------------ + +The process is expected to respond with a list of "key=value" pairs +terminated with a flush packet. If the process does not experience +problems then the list must contain a "success" status. +------------------------ +packet: git< status=success +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content, it +is expected to respond with an "error" status. +------------------------ +packet: git< status=error +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content as +well as any future content for the lifetime of the Git process, then it +is expected to respond with an "abort" status at any point in the +protocol. +------------------------ +packet: git< status=abort +packet: git< 0000 +------------------------ + +Git neither stops nor restarts the process in case the "error"/"abort" +status is set. + +If the process dies during the communication or does not adhere to the +protocol then Git will stop the process and restart it with the next +object that needs to be processed. + +After the read-object process has processed an object it is expected to +wait for the next "key=value" list containing a command. Git will close +the command pipe on exit. The process is expected to detect EOF and exit +gracefully on its own. Git will wait until the process has stopped. + +A long running read-object process demo implementation can be found in +`contrib/long-running-read-object/example.pl` located in the Git core +repository. If you develop your own long running process then the +`GIT_TRACE_PACKET` environment variables can be very helpful for +debugging (see linkgit:git[1]). diff --git a/Documentation/technical/sparse-index.adoc b/Documentation/technical/sparse-index.adoc index 3b24c1a219f811..c466dbddc930a9 100644 --- a/Documentation/technical/sparse-index.adoc +++ b/Documentation/technical/sparse-index.adoc @@ -206,3 +206,10 @@ Here are some commands that might be useful to update: * `git am` * `git clean` * `git stash` + +In order to help identify the cases where remaining index expansion is +occurring in user machines, calls to `ensure_full_index()` have been +replaced with `ensure_full_index_with_reason()` or with +`ensure_full_index_unaudited()`. These versions add tracing that should +help identify the reason for the index expansion without needing full +access to someone's repository. diff --git a/Documentation/technical/status-serialization-format.txt b/Documentation/technical/status-serialization-format.txt new file mode 100644 index 00000000000000..475ae814495581 --- /dev/null +++ b/Documentation/technical/status-serialization-format.txt @@ -0,0 +1,107 @@ +Git status serialization format +=============================== + +Git status serialization enables git to dump the results of a status scan +to a binary file. This file can then be loaded by later status invocations +to print the cached status results. + +The file contains the essential fields from: +() the index +() the "struct wt_status" for the overall results +() the contents of "struct wt_status_change_data" for tracked changed files +() the list of untracked and ignored files + +Version 1 Format: +================= + +The V1 file begins with a required header section followed by optional +sections for each type of item (changed, untracked, ignored). Individual +item sections are only present if necessary. Each item section begins +with an item-type header with the number of items in the section. + +Each "line" in the format is encoded using pkt-line with a final LF. +Flush packets are used to terminate sections. + +----------------- +PKT-LINE("version" SP "1") + +[] +[] +[] +----------------- + + +V1 Header +--------- + +The v1-header-section fields are taken directly from "struct wt_status". +Each field is printed on a separate pkt-line. Lines for NULL string +values are omitted. All integers are printed with "%d". OIDs are +printed in hex. + +v1-header-section = + + PKT-LINE() + +v1-index-headers = PKT-LINE("index_mtime" SP SP LF) + +v1-wt-status-headers = PKT-LINE("is_initial" SP LF) + [ PKT-LINE("branch" SP LF) ] + [ PKT-LINE("reference" SP LF) ] + PKT-LINE("show_ignored_files" SP LF) + PKT-LINE("show_untracked_files" SP LF) + PKT-LINE("show_ignored_directory" SP LF) + [ PKT-LINE("ignore_submodule_arg" SP LF) ] + PKT-LINE("detect_rename" SP LF) + PKT-LINE("rename_score" SP LF) + PKT-LINE("rename_limit" SP LF) + PKT-LINE("detect_break" SP LF) + PKT-LINE("sha1_commit" SP LF) + PKT-LINE("committable" SP LF) + PKT-LINE("workdir_dirty" SP LF) + + +V1 Changed Items +---------------- + +The v1-changed-item-section lists all of the changed items with one +item per pkt-line. Each pkt-line contains: a binary block of data +from "struct wt_status_serialize_data_fixed" in a fixed header where +integers are in network byte order and OIDs are in raw (non-hex) form. +This is followed by one or two raw pathnames (not c-quoted) with NUL +terminators (both NULs are always present even if there is no rename). + +v1-changed-item-section = PKT-LINE("changed" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +changed_item = + + + + + + + + + + + + NUL + [ ] + NUL + + +V1 Untracked and Ignored Items +------------------------------ + +These sections are simple lists of pathnames. They ARE NOT +c-quoted. + +v1-untracked-item-section = PKT-LINE("untracked" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +v1-ignored-item-section = PKT-LINE("ignored" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() diff --git a/Documentation/trace2-target-values.adoc b/Documentation/trace2-target-values.adoc index 06f19533134f9d..0e035b0708dac4 100644 --- a/Documentation/trace2-target-values.adoc +++ b/Documentation/trace2-target-values.adoc @@ -1,12 +1,12 @@ --- +---- * `0` or `false` - Disables the target. * `1` or `true` - Writes to `STDERR`. * `[2-9]` - Writes to the already opened file descriptor. * `` - Writes to the file in append mode. If the target -already exists and is a directory, the traces will be written to files (one -per process) underneath the given directory. + already exists and is a directory, the traces will be written to files (one + per process) underneath the given directory. * `af_unix:[:]` - Write to a -Unix DomainSocket (on platforms that support them). Socket -type can be either `stream` or `dgram`; if omitted Git will -try both. --- + Unix DomainSocket (on platforms that support them). Socket + type can be either `stream` or `dgram`; if omitted Git will + try both. +---- diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 3b5fe08c19cc40..9470b510a63ed7 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -2,6 +2,9 @@ DEF_VER=v2.56.0-rc2 +# Identify microsoft/git via a distinct version suffix +DEF_VER=$DEF_VER.vfs.0.0 + LF=' ' @@ -47,9 +50,15 @@ then test -d "${GIT_DIR:-.git}" || test -f "$SOURCE_DIR"/.git; } && - VN=$(git -C "$SOURCE_DIR" describe --dirty --match="v[0-9]*" 2>/dev/null) && + VN=$(git -C "$SOURCE_DIR" describe --dirty --match="v[0-9]*vfs*" 2>/dev/null) && case "$VN" in *$LF*) (exit 1) ;; + v[0-9]*) + if test "${VN%%.vfs.*}" != "${DEF_VER%%.vfs.*}" + then + echo "Found version $VN, which is not based on $DEF_VER" >&2 + exit 1 + fi ;; esac then VN=$(echo "$VN" | sed -e 's/-/./g'); diff --git a/Makefile b/Makefile index c98be9155c54ae..633c1170531c7a 100644 --- a/Makefile +++ b/Makefile @@ -1187,6 +1187,8 @@ LIB_OBJS += git-zlib.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o +LIB_OBJS += gvfs.o +LIB_OBJS += gvfs-helper-client.o LIB_OBJS += hash-lookup.o LIB_OBJS += hash.o LIB_OBJS += hashmap.o @@ -1377,6 +1379,7 @@ LIB_OBJS += varint.o endif LIB_OBJS += version.o LIB_OBJS += versioncmp.o +LIB_OBJS += virtualfilesystem.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += worktree.o @@ -1384,6 +1387,8 @@ LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o LIB_OBJS += wt-status.o +LIB_OBJS += wt-status-deserialize.o +LIB_OBJS += wt-status-serialize.o LIB_OBJS += xdiff-interface.o LIB_OBJS += xdiff/xdiffi.o LIB_OBJS += xdiff/xemit.o @@ -1515,6 +1520,7 @@ BUILTIN_OBJS += builtin/tag.o BUILTIN_OBJS += builtin/unpack-file.o BUILTIN_OBJS += builtin/unpack-objects.o BUILTIN_OBJS += builtin/update-index.o +BUILTIN_OBJS += builtin/update-microsoft-git.o BUILTIN_OBJS += builtin/update-ref.o BUILTIN_OBJS += builtin/update-server-info.o BUILTIN_OBJS += builtin/upload-archive.o @@ -1836,6 +1842,9 @@ endif endif BASIC_CFLAGS += $(CURL_CFLAGS) + PROGRAM_OBJS += gvfs-helper.o + TEST_PROGRAMS_NEED_X += test-gvfs-protocol + REMOTE_CURL_PRIMARY = git-remote-http$X REMOTE_CURL_ALIASES = git-remote-https$X git-remote-ftp$X git-remote-ftps$X REMOTE_CURL_NAMES = $(REMOTE_CURL_PRIMARY) $(REMOTE_CURL_ALIASES) @@ -2899,6 +2908,7 @@ GIT_OBJS += git.o .PHONY: git-objs git-objs: $(GIT_OBJS) +SCALAR_OBJS := json-parser.o SCALAR_OBJS += scalar.o .PHONY: scalar-objs scalar-objs: $(SCALAR_OBJS) @@ -3006,7 +3016,7 @@ gettext.sp gettext.s gettext.o: GIT-PREFIX gettext.sp gettext.s gettext.o: EXTRA_CPPFLAGS = \ -DGIT_LOCALE_PATH='"$(localedir_relative_SQ)"' -http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp: SP_EXTRA_FLAGS += \ +http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp gvfs-helper.sp: SP_EXTRA_FLAGS += \ -DCURL_DISABLE_TYPECHECK pack-revindex.sp: SP_EXTRA_FLAGS += -Wno-memcpy-max-count @@ -3051,10 +3061,14 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OB $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) -scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS) +scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \ $(filter %.o,$^) $(LIBS) +git-gvfs-helper$X: gvfs-helper.o http.o GIT-LDFLAGS $(GITLIBS) $(LAZYLOAD_LIBCURL_OBJ) + $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ + $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) + $(LIB_FILE): $(LIB_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ @@ -3853,7 +3867,7 @@ dist: git-archive$(X) configure @$(MAKE) -C git-gui TARDIR=../.dist-tmp-dir/git-gui dist-version ./git-archive --format=tar \ $(GIT_ARCHIVE_EXTRA_FILES) \ - --prefix=$(GIT_TARNAME)/ HEAD^{tree} > $(GIT_TARNAME).tar + --prefix=$(GIT_TARNAME)/ HEAD > $(GIT_TARNAME).tar @$(RM) -r .dist-tmp-dir gzip -f -9 $(GIT_TARNAME).tar diff --git a/README.md b/README.md index 026d5d85caef09..3e0fa1516b879c 100644 --- a/README.md +++ b/README.md @@ -1,148 +1,221 @@ -Git for Windows -=============== - -[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) -[![Open in Visual Studio Code](https://img.shields.io/static/v1?logo=visualstudiocode&label=&message=Open%20in%20Visual%20Studio%20Code&labelColor=2c2c32&color=007acc&logoColor=007acc)](https://open.vscode.dev/git-for-windows/git) -[![Build status](https://github.com/git-for-windows/git/workflows/CI/badge.svg)](https://github.com/git-for-windows/git/actions?query=branch%3Amain+event%3Apush) -[![Join the chat at https://gitter.im/git-for-windows/git](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/git-for-windows/git?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - -This is [Git for Windows](http://git-for-windows.github.io/), the Windows port -of [Git](http://git-scm.com/). - -The Git for Windows project is run using a [governance -model](http://git-for-windows.github.io/governance-model.html). If you -encounter problems, you can report them as [GitHub -issues](https://github.com/git-for-windows/git/issues), discuss them in Git -for Windows' [Discussions](https://github.com/git-for-windows/git/discussions) -or on the [Git mailing list](mailto:git@vger.kernel.org), and [contribute bug -fixes](https://gitforwindows.org/how-to-participate). - -To build Git for Windows, please either install [Git for Windows' -SDK](https://gitforwindows.org/#download-sdk), start its `git-bash.exe`, `cd` -to your Git worktree and run `make`, or open the Git worktree as a folder in -Visual Studio. - -To verify that your build works, use one of the following methods: - -- If you want to test the built executables within Git for Windows' SDK, - prepend `/bin-wrappers` to the `PATH`. -- Alternatively, run `make install` in the Git worktree. -- If you need to test this in a full installer, run `sdk build - git-and-installer`. -- You can also "install" Git into an existing portable Git via `make install - DESTDIR=` where `` refers to the top-level directory of the - portable Git. In this instance, you will want to prepend that portable Git's - `/cmd` directory to the `PATH`, or test by running that portable Git's - `git-bash.exe` or `git-cmd.exe`. -- If you built using a recent Visual Studio, you can use the menu item - `Build>Install git` (you will want to click on `Project>CMake Settings for - Git` first, then click on `Edit JSON` and then point `installRoot` to the - `mingw64` directory of an already-unpacked portable Git). - - As in the previous bullet point, you will then prepend `/cmd` to the `PATH` - or run using the portable Git's `git-bash.exe` or `git-cmd.exe`. -- If you want to run the built executables in-place, but in a CMD instead of - inside a Bash, you can run a snippet like this in the `git-bash.exe` window - where Git was built (ensure that the `EOF` line has no leading spaces), and - then paste into the CMD window what was put in the clipboard: - - ```sh - clip.exe < -including full documentation and Git related tools. - -See [Documentation/gittutorial.adoc][] to get started, then see -[Documentation/giteveryday.adoc][] for a useful minimum set of commands, and -`Documentation/git-.adoc` for documentation of each command. -If git has been correctly installed, then the tutorial can also be -read with `man gittutorial` or `git help tutorial`, and the -documentation of each command with `man git-` or `git help -`. - -CVS users may also want to read [Documentation/gitcvs-migration.adoc][] -(`man gitcvs-migration` or `git help cvs-migration` if git is -installed). - -The user discussion and development of core Git take place on the Git -mailing list -- everyone is welcome to post bug reports, feature -requests, comments and patches to git@vger.kernel.org (read -[Documentation/SubmittingPatches][] for instructions on patch submission -and [Documentation/CodingGuidelines][]). - -Those wishing to help with error message, usage and informational message -string translations (localization l10) should see [po/README.md][] -(a `po` file is a Portable Object file that holds the translations). - -To subscribe to the list, send an email to -(see https://subspace.kernel.org/subscribing.html for details). The mailing -list archives are available at , - and other archival sites. -The core git mailing list is plain text (no HTML!). - -Issues which are security relevant should be disclosed privately to -the Git Security mailing list . - -The maintainer frequently sends the "What's cooking" reports that -list the current status of various development topics to the mailing -list. The discussion following them give a good reference for -project status, development direction and remaining tasks. - -The name "git" was given by Linus Torvalds when he wrote the very -first version. He described the tool as "the stupid content tracker" -and the name as (depending on your mood): - - - random three-letter combination that is pronounceable, and not - actually used by any common UNIX command. The fact that it is a - mispronunciation of "get" may or may not be relevant. - - stupid. contemptible and despicable. simple. Take your pick from the - dictionary of slang. - - "global information tracker": you're in a good mood, and it actually - works for you. Angels sing, and a light suddenly fills the room. - - "goddamn idiotic truckload of sh*t": when it breaks - -[INSTALL]: INSTALL -[Documentation/gittutorial.adoc]: Documentation/gittutorial.adoc -[Documentation/giteveryday.adoc]: Documentation/giteveryday.adoc -[Documentation/gitcvs-migration.adoc]: Documentation/gitcvs-migration.adoc -[Documentation/SubmittingPatches]: Documentation/SubmittingPatches -[Documentation/CodingGuidelines]: Documentation/CodingGuidelines -[po/README.md]: po/README.md +If you're working in a monorepo and want to take advantage of the performance boosts in +`microsoft/git`, then you can download the latest version installer for your OS from the +[Releases page](https://github.com/microsoft/git/releases). Alternatively, you can opt to install +via the command line, using the below instructions for supported OSes: + +## Windows + +__Note:__ Winget is still in public preview, meaning you currently +[need to take special installation steps](https://docs.microsoft.com/en-us/windows/package-manager/winget/#install-winget): +Either manually install the `.appxbundle` available at the +[preview version of App Installer](https://www.microsoft.com/p/app-installer/9nblggh4nns1?ocid=9nblggh4nns1_ORSEARCH_Bing&rtc=1&activetab=pivot:overviewtab), +or participate in the +[Windows Insider flight ring](https://insider.windows.com/https://insider.windows.com/) +since `winget` is available by default on preview versions of Windows. + +To install with Winget, run + +```shell +winget install --id microsoft.git +``` + +Double-check that you have the right version by running these commands, +which should have the same output: + +```shell +git version +scalar version +``` + +To upgrade `microsoft/git`, use the following Git command, which will download and install the latest +release. + +```shell +git update-microsoft-git +``` + +You may also be alerted with a notification to upgrade, which presents a single-click process for +running `git update-microsoft-git`. + +## macOS + +To install `microsoft/git` on macOS, first [be sure that Homebrew is installed](https://brew.sh/) then +install the `microsoft-git` cask with these steps: + +```shell +brew tap microsoft/git +brew install --cask microsoft-git +``` + +Double-check that you have the right version by running these commands, +which should have the same output: + +```shell +git version +scalar version +``` + +To upgrade microsoft/git, you can run the necessary `brew` commands: + +```shell +brew update +brew upgrade --cask microsoft-git +``` + +Or you can run the `git update-microsoft-git` command, which will run those brew commands for you. + +## Linux +### Ubuntu/Debian distributions + +On newer distributions*, you can install using the most recent Debian package. +To download and validate the signature of this package, run the following: + +```shell +# Install needed packages +sudo apt-get install -y curl debsig-verify + +# Download public key signature file +curl -Os https://packages.microsoft.com/keys/microsoft-2025.asc + +# De-armor public key signature file +gpg --output microsoft-2025.gpg --dearmor microsoft-2025.asc + +# Note that the fingerprint of this key is "EE4D7792F748182B", which you can +# determine by running: +gpg --show-keys microsoft-2025.asc | head -n 2 | tail -n 1 | tail -c 17 + +# Copy de-armored public key to debsig keyring folder +sudo mkdir /usr/share/debsig/keyrings/EE4D7792F748182B +sudo mv microsoft-2025.gpg /usr/share/debsig/keyrings/EE4D7792F748182B/ + +# Create an appropriate policy file +sudo mkdir /etc/debsig/policies/EE4D7792F748182B +cat > generic.pol << EOL + + + + + + + + + + + +EOL + +sudo mv generic.pol /etc/debsig/policies/EE4D7792F748182B/generic.pol + +# Download Debian package (substitute `amd64` with `arm64` on ARM machines) +curl -s https://api.github.com/repos/microsoft/git/releases/latest \ +| grep "browser_download_url.*amd64.deb" \ +| cut -d : -f 2,3 \ +| tr -d \" \ +| xargs -I 'url' curl -L -o msft-git.deb 'url' + +# Verify +debsig-verify msft-git.deb + +# Install +sudo dpkg -i msft-git.deb +``` + +Double-check that you have the right version by running these commands, +which should have the same output: + +```shell +git version +scalar version +``` + +To upgrade, you will need to repeat these steps to reinstall. + +*Older distributions are missing some required dependencies. Even +though the package may appear to install successfully, `microsoft/ +git` will not function as expected. If you are running `Ubuntu 20.04` or +older, please follow the install from source instructions below +instead of installing the debian package. + +### Installing From Source + +On older or other distros you will need to compile and install `microsoft/git` from source: + +```shell +git clone https://github.com/microsoft/git microsoft-git +cd microsoft-git +make -j12 prefix=/usr/local +sudo make -j12 prefix=/usr/local install +``` + +For more assistance building Git from source, see +[the INSTALL file in the core Git project](https://github.com/git/git/blob/master/INSTALL). + +#### Common Debian based dependencies +While the INSTALL file covers dependencies in detail, here is a shortlist of common required dependencies on older Debian/Ubuntu distros: + +```shell +sudo apt-get update +sudo apt-get install libz-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext cmake gcc +``` + +Contributing +========================================================= + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit . + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/abspath.c b/abspath.c index 0c17e98654e4b0..e899f46d02097a 100644 --- a/abspath.c +++ b/abspath.c @@ -14,7 +14,7 @@ int is_directory(const char *path) } /* removes the last path component from 'path' except if 'path' is root */ -static void strip_last_component(struct strbuf *path) +void strip_last_path_component(struct strbuf *path) { size_t offset = offset_1st_component(path->buf); size_t len = path->len; @@ -119,7 +119,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path, continue; /* '.' component */ } else if (next.len == 2 && !strcmp(next.buf, "..")) { /* '..' component; strip the last path component */ - strip_last_component(resolved); + strip_last_path_component(resolved); continue; } @@ -171,7 +171,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path, * strip off the last component since it will * be replaced with the contents of the symlink */ - strip_last_component(resolved); + strip_last_path_component(resolved); } /* diff --git a/abspath.h b/abspath.h index 4653080d5e4b7a..06241ba13cf646 100644 --- a/abspath.h +++ b/abspath.h @@ -10,6 +10,11 @@ char *real_pathdup(const char *path, int die_on_error); const char *absolute_path(const char *path); char *absolute_pathdup(const char *path); +/** + * Remove the last path component from 'path' except if 'path' is root. + */ +void strip_last_path_component(struct strbuf *path); + /* * Concatenate "prefix" (if len is non-zero) and "path", with no * connecting characters (so "prefix" should end with a "/"). diff --git a/advice.c b/advice.c index 3fa240c521494c..c79f2436a65c5b 100644 --- a/advice.c +++ b/advice.c @@ -91,9 +91,13 @@ static struct { [ADVICE_SUBMODULE_MERGE_CONFLICT] = { "submoduleMergeConflict" }, [ADVICE_SUGGEST_DETACHING_HEAD] = { "suggestDetachingHead" }, [ADVICE_UPDATE_SPARSE_PATH] = { "updateSparsePath" }, + [ADVICE_USE_CORE_CONFIG_WRITE_LOCK_TIMEOUT_MS_CONFIG] = { "useCoreConfigWriteLockTimeoutMSConfig" }, [ADVICE_USE_CORE_FSMONITOR_CONFIG] = { "useCoreFSMonitorConfig" }, [ADVICE_WAITING_FOR_EDITOR] = { "waitingForEditor" }, [ADVICE_WORKTREE_ADD_ORPHAN] = { "worktreeAddOrphan" }, + + /* microsoft/git custom advice below: */ + [ADVICE_GVFS_HELPER_TRANSIENT_RETRY] = { "gvfs.transientRetry"}, }; static const char turn_off_instructions[] = diff --git a/advice.h b/advice.h index ba02040196dbeb..15321252aaa411 100644 --- a/advice.h +++ b/advice.h @@ -58,9 +58,13 @@ enum advice_type { ADVICE_SUBMODULE_MERGE_CONFLICT, ADVICE_SUGGEST_DETACHING_HEAD, ADVICE_UPDATE_SPARSE_PATH, + ADVICE_USE_CORE_CONFIG_WRITE_LOCK_TIMEOUT_MS_CONFIG, ADVICE_USE_CORE_FSMONITOR_CONFIG, ADVICE_WAITING_FOR_EDITOR, ADVICE_WORKTREE_ADD_ORPHAN, + + /* microsoft/git custom advice below: */ + ADVICE_GVFS_HELPER_TRANSIENT_RETRY, }; int git_default_advice_config(const char *var, const char *value); diff --git a/apply.c b/apply.c index f7f9f21809df5f..19a7ab55af0900 100644 --- a/apply.c +++ b/apply.c @@ -20,6 +20,7 @@ #include "dir.h" #include "environment.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "xdiff-interface.h" #include "merge-ll.h" @@ -3499,6 +3500,25 @@ static int checkout_target(struct index_state *istate, { struct checkout costate = CHECKOUT_INIT; + /* + * Do not checkout the entry if the skipworktree bit is set + * + * Both callers of this method (check_preimage and load_current) + * check for the existance of the file before calling this + * method so we know that the file doesn't exist at this point + * and we don't need to perform that check again here. + * We just need to check the skip-worktree and return. + * + * This is to prevent git from creating a file in the + * working directory that has the skip-worktree bit on, + * then updating the index from the patch and not keeping + * the working directory version up to date with what it + * changed the index version to be. + */ + if (gvfs_config_is_set(istate->repo, GVFS_USE_VIRTUAL_FILESYSTEM) && + ce_skip_worktree(ce)) + return 0; + costate.refresh_cache = 1; costate.istate = istate; if (checkout_entry(ce, &costate, NULL, NULL) || diff --git a/bin-wrappers/.gitignore b/bin-wrappers/.gitignore index 1c6c90458b7586..e481f5a45a7a0d 100644 --- a/bin-wrappers/.gitignore +++ b/bin-wrappers/.gitignore @@ -6,4 +6,5 @@ /git-upload-pack /scalar /test-fake-ssh +/test-gvfs-protocol /test-tool diff --git a/branch.c b/branch.c index 22f4f46b96ed3a..eef06b4713a374 100644 --- a/branch.c +++ b/branch.c @@ -224,6 +224,8 @@ static int inherit_tracking(struct tracking *tracking, const char *orig_ref) skip_prefix(orig_ref, "refs/heads/", &bare_ref); branch = branch_get(bare_ref); + if (!branch) + BUG("could not get branch for '%s", bare_ref); if (!branch->remote_name) { warning(_("asked to inherit tracking from '%s', but no remote is set"), bare_ref); diff --git a/builtin.h b/builtin.h index d3caec75424f71..3fca497462021e 100644 --- a/builtin.h +++ b/builtin.h @@ -268,6 +268,7 @@ int cmd_tag(int argc, const char **argv, const char *prefix, struct repository * int cmd_unpack_file(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_unpack_objects(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_update_index(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_update_microsoft_git(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_update_ref(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_update_server_info(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_upload_archive(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/add.c b/builtin/add.c index 31cf7daaa4c155..5517622aa561fe 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -1,3 +1,5 @@ +#define USE_THE_REPOSITORY_VARIABLE + /* * "git add" builtin command * @@ -5,6 +7,7 @@ */ #include "builtin.h" +#include "environment.h" #include "advice.h" #include "config.h" #include "environment.h" @@ -52,6 +55,7 @@ static int chmod_pathspec(struct repository *repo, int err; if (!include_sparse && + !core_virtualfilesystem && (ce_skip_worktree(ce) || !path_in_sparse_checkout(ce->name, repo->index))) continue; @@ -137,8 +141,9 @@ static int refresh(struct repository *repo, int verbose, const struct pathspec * if (!seen[i]) { const char *path = pathspec->items[i].original; - if (matches_skip_worktree(pathspec, i, &skip_worktree_seen) || - !path_in_sparse_checkout(path, repo->index)) { + if (!core_virtualfilesystem && + (matches_skip_worktree(pathspec, i, &skip_worktree_seen) || + !path_in_sparse_checkout(path, repo->index))) { string_list_append(&only_match_skip_worktree, pathspec->items[i].original); } else { @@ -148,7 +153,11 @@ static int refresh(struct repository *repo, int verbose, const struct pathspec * } } - if (only_match_skip_worktree.nr) { + /* + * When using a virtual filesystem, we might re-add a path + * that is currently virtual and we want that to succeed. + */ + if (!core_virtualfilesystem && only_match_skip_worktree.nr) { advise_on_updating_sparse_paths(&only_match_skip_worktree); ret = 1; } @@ -633,7 +642,11 @@ int cmd_add(int argc, if (seen[i]) continue; - if (!include_sparse && + /* + * When using a virtual filesystem, we might re-add a path + * that is currently virtual and we want that to succeed. + */ + if (!include_sparse && !core_virtualfilesystem && matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) { string_list_append(&only_match_skip_worktree, pathspec.items[i].original); @@ -657,7 +670,6 @@ int cmd_add(int argc, } } - if (only_match_skip_worktree.nr) { advise_on_updating_sparse_paths(&only_match_skip_worktree); exit_status = 1; diff --git a/builtin/am.c b/builtin/am.c index e9623b8307793f..6b38af21228cdd 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -434,20 +434,20 @@ static void am_load(struct am_state *state) } read_state_file(&sb, state, "keep", 1); - if (!strcmp(sb.buf, "t")) + if (!strcmp(sb.buf, "t")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->keep = KEEP_TRUE; - else if (!strcmp(sb.buf, "b")) + else if (!strcmp(sb.buf, "b")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->keep = KEEP_NON_PATCH; else state->keep = KEEP_FALSE; read_state_file(&sb, state, "messageid", 1); - state->message_id = !strcmp(sb.buf, "t"); + state->message_id = !strcmp(sb.buf, "t"); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand read_state_file(&sb, state, "scissors", 1); - if (!strcmp(sb.buf, "t")) + if (!strcmp(sb.buf, "t")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->scissors = SCISSORS_TRUE; - else if (!strcmp(sb.buf, "f")) + else if (!strcmp(sb.buf, "f")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->scissors = SCISSORS_FALSE; else state->scissors = SCISSORS_UNSET; @@ -455,12 +455,12 @@ static void am_load(struct am_state *state) read_state_file(&sb, state, "quoted-cr", 1); if (!*sb.buf) state->quoted_cr = quoted_cr_unset; - else if (mailinfo_parse_quoted_cr_action(sb.buf, &state->quoted_cr) != 0) + else if (mailinfo_parse_quoted_cr_action(sb.buf, &state->quoted_cr) != 0) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die(_("could not parse %s"), am_path(state, "quoted-cr")); read_state_file(&sb, state, "apply-opt", 1); strvec_clear(&state->git_apply_opts); - if (sq_dequote_to_strvec(sb.buf, &state->git_apply_opts) < 0) + if (sq_dequote_to_strvec(sb.buf, &state->git_apply_opts) < 0) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die(_("could not parse %s"), am_path(state, "apply-opt")); state->rebasing = !!file_exists(am_path(state, "rebasing")); diff --git a/builtin/blame.c b/builtin/blame.c index 48d5251c6df700..00bbdc6bb9d4a2 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -64,6 +64,7 @@ static int incremental; static int xdl_opts; static int abbrev = -1; static int no_whole_file_rename; +static int blame_detect_rename = -1; static int show_progress; static char repeated_meta_color[COLOR_MAXLEN]; static int coloring_mode; @@ -809,6 +810,27 @@ static int git_blame_config(const char *var, const char *value, } } + if (!strcmp(var, "blame.renames")) { + blame_detect_rename = git_config_bool(var, value); + return 0; + } + + /* + * Blame does not use git_diff_basic_config in its config + * chain, so diff_rename_score_default is not normally loaded. + * Forward blame.renameThreshold as diff.renameThreshold to + * set the global that repo_diff_setup() copies into + * diff_options.rename_score. + */ + if (!strcmp(var, "blame.renamethreshold")) + return git_diff_basic_config("diff.renamethreshold", + value, ctx, cb); + + /* Same approach for blame.renameLimit; see above. */ + if (!strcmp(var, "blame.renamelimit")) + return git_diff_basic_config("diff.renamelimit", + value, ctx, cb); + if (!strcmp(var, "diff.algorithm")) { long diff_algorithm; if (!value) @@ -1059,7 +1081,10 @@ int cmd_blame(int argc, } parse_done: revision_opts_finish(&revs); - no_whole_file_rename = !revs.diffopt.flags.follow_renames; + if (blame_detect_rename >= 0) + no_whole_file_rename = !blame_detect_rename; + if (!revs.diffopt.flags.follow_renames) + no_whole_file_rename = 1; xdl_opts |= revs.diffopt.xdl_opts & XDF_INDENT_HEURISTIC; revs.diffopt.flags.follow_renames = 0; argc = parse_options_end(&ctx); diff --git a/builtin/cat-file.c b/builtin/cat-file.c index e8fdf1a51f5caf..7037f9a84e87e2 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -136,7 +136,7 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name) struct object_id oid; enum object_type type; char *buf; - size_t size; + size_t size = 0; struct object_context obj_context = {0}; struct object_info oi = OBJECT_INFO_INIT; unsigned flags = OBJECT_INFO_LOOKUP_REPLACE; diff --git a/builtin/checkout-index.c b/builtin/checkout-index.c index 311b94ff3174a6..5ff73327858b7e 100644 --- a/builtin/checkout-index.c +++ b/builtin/checkout-index.c @@ -156,7 +156,7 @@ static int checkout_all(struct index_state *index, const char *prefix, int prefi * first entry inside the expanded sparse directory). */ if (ignore_skip_worktree) { - ensure_full_index(index); + ensure_full_index_with_reason(index, "checkout-index"); ce = index->cache[i]; } } diff --git a/builtin/checkout.c b/builtin/checkout.c index f2e5a3308e37f7..d23550a126be42 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -20,6 +20,7 @@ #include "object-file.h" #include "object-name.h" #include "odb.h" +#include "packfile.h" #include "parse-options.h" #include "path.h" #include "preload-index.h" @@ -226,6 +227,24 @@ static int update_some(const struct object_id *oid, struct strbuf *base, discard_cache_entry(ce); return 0; } + + /* + * When a virtual filesystem is in use, preserve + * skip-worktree from the existing index entry. + * Without this, checkout_entry() would try to + * unlink() and recreate the file on disk, but + * virtual (projected) files have no physical NTFS + * entry and the unlink fails with ENOENT, causing + * the checkout to fail with exit code 255. + * + * Preserving skip-worktree lets the index update to + * the new tree entry's OID while skipping the + * working tree write. The virtual filesystem + * provider will serve the correct content from the + * updated projection on next access. + */ + if (core_virtualfilesystem && ce_skip_worktree(old)) + ce->ce_flags |= CE_SKIP_WORKTREE; } add_index_entry(the_repository->index, ce, @@ -389,7 +408,18 @@ static void mark_ce_for_checkout_overlay(struct cache_entry *ce, const struct checkout_opts *opts) { ce->ce_flags &= ~CE_MATCHED; - if (!opts->ignore_skipworktree && ce_skip_worktree(ce)) + if (!opts->ignore_skipworktree && ce_skip_worktree(ce) && + !(core_virtualfilesystem && opts->source_tree && + (ce->ce_flags & CE_UPDATE))) + /* + * Skip-worktree entries are normally excluded from + * pathspec matching. The exception is virtual + * filesystem entries updated from a source tree + * (CE_UPDATE set by update_some): those must still + * match so report_path_error() does not reject them. + * The actual worktree write is skipped later in + * checkout_worktree() because skip-worktree is set. + */ return; if (opts->source_tree && !(ce->ce_flags & CE_UPDATE)) /* @@ -467,6 +497,9 @@ static int checkout_worktree(const struct checkout_opts *opts, struct cache_entry *ce = the_repository->index->cache[pos]; if (ce->ce_flags & CE_MATCHED) { if (!ce_stage(ce)) { + if (core_virtualfilesystem && + ce_skip_worktree(ce)) + continue; errs |= checkout_entry(ce, &state, NULL, &nr_checkouts); continue; @@ -1050,8 +1083,16 @@ static void update_refs_for_switch(const struct checkout_opts *opts, strbuf_release(&msg); if (!opts->quiet && !opts->force_detach && - (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) + (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) { + unsigned long nr_unpack_entry_at_start; + + trace2_region_enter("tracking", "report_tracking", the_repository); + nr_unpack_entry_at_start = get_nr_unpack_entry(); report_tracking(new_branch_info); + trace2_data_intmax("tracking", NULL, "report_tracking/nr_unpack_entries", + (intmax_t)(get_nr_unpack_entry() - nr_unpack_entry_at_start)); + trace2_region_leave("tracking", "report_tracking", the_repository); + } } static int add_pending_uninteresting_ref(const struct reference *ref, void *cb_data) diff --git a/builtin/clone.c b/builtin/clone.c index 5b25cca5102956..690da1a09bdb66 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -119,7 +119,7 @@ static const char *get_repo_path_1(struct strbuf *path, int *is_bundle) continue; len = read_in_full(fd, signature, 8); close(fd); - if (len != 8 || strncmp(signature, "gitdir: ", 8)) + if (len != 8 || strncmp(signature, "gitdir: ", 8)) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand continue; dst = read_gitfile(path->buf); if (dst) { diff --git a/builtin/commit.c b/builtin/commit.c index 1519644aae035c..2929913d508d82 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -43,6 +43,7 @@ #include "commit-graph.h" #include "pretty.h" #include "trailer.h" +#include "trace2.h" static const char * const builtin_commit_usage[] = { N_("git commit [-a | --interactive | --patch] [-s] [-v] [-u[]] [--amend]\n" @@ -166,6 +167,122 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un return 0; } +static int do_serialize = 0; +static char *serialize_path = NULL; + +static int reject_implicit = 0; +static int do_implicit_deserialize = 0; +static int do_explicit_deserialize = 0; +static char *deserialize_path = NULL; + +static enum wt_status_deserialize_wait implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; +static enum wt_status_deserialize_wait explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + +/* + * --serialize | --serialize= + * + * Request that we serialize status output rather than or in addition to + * printing in any of the established formats. + * + * Without a path, we write binary serialization data to stdout (and omit + * the normal status output). + * + * With a path, we write binary serialization data to the and then + * write normal status output. + */ +static int opt_parse_serialize(const struct option *opt, const char *arg, int unset) +{ + enum wt_status_format *value = (enum wt_status_format *)opt->value; + if (unset || !arg) + *value = STATUS_FORMAT_SERIALIZE_V1; + + if (arg) { + free(serialize_path); + serialize_path = xstrdup(arg); + } + + if (do_explicit_deserialize) + die("cannot mix --serialize and --deserialize"); + do_implicit_deserialize = 0; + + do_serialize = 1; + return 0; +} + +/* + * --deserialize | --deserialize= | + * --no-deserialize + * + * Request that we deserialize status data from some existing resource + * rather than performing a status scan. + * + * The input source can come from stdin or a path given here -- or be + * inherited from the config settings. + */ +static int opt_parse_deserialize(const struct option *opt UNUSED, const char *arg, int unset) +{ + if (unset) { + do_implicit_deserialize = 0; + do_explicit_deserialize = 0; + } else { + if (do_serialize) + die("cannot mix --serialize and --deserialize"); + if (arg) { + /* override config or stdin */ + free(deserialize_path); + deserialize_path = xstrdup(arg); + } + if (!deserialize_path || !*deserialize_path) + do_explicit_deserialize = 1; /* read stdin */ + else if (wt_status_deserialize_access(deserialize_path, R_OK) == 0) + do_explicit_deserialize = 1; /* can read from this file */ + else { + /* + * otherwise, silently fallback to the normal + * collection scan + */ + do_implicit_deserialize = 0; + do_explicit_deserialize = 0; + } + } + + return 0; +} + +static enum wt_status_deserialize_wait parse_dw(const char *arg) +{ + int tenths; + + if (!strcmp(arg, "fail")) + return DESERIALIZE_WAIT__FAIL; + else if (!strcmp(arg, "block")) + return DESERIALIZE_WAIT__BLOCK; + else if (!strcmp(arg, "no")) + return DESERIALIZE_WAIT__NO; + + /* + * Otherwise, assume it is a timeout in tenths of a second. + * If it contains a bogus value, atol() will return zero + * which is OK. + */ + tenths = atol(arg); + if (tenths < 0) + tenths = DESERIALIZE_WAIT__NO; + return tenths; +} + +static int opt_parse_deserialize_wait(const struct option *opt UNUSED, + const char *arg, + int unset) +{ + if (unset) + explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + else + explicit_deserialize_wait = parse_dw(arg); + + return 0; +} + static int opt_parse_m(const struct option *opt, const char *arg, int unset) { struct strbuf *buf = opt->value; @@ -270,7 +387,7 @@ static int list_paths(struct string_list *list, const char *with_tree, } /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(the_repository->index); + ensure_full_index_unaudited(the_repository->index); for (i = 0; i < the_repository->index->cache_nr; i++) { const struct cache_entry *ce = the_repository->index->cache[i]; struct string_list_item *item; @@ -1051,7 +1168,7 @@ static int prepare_to_commit(const char *index_file, const char *prefix, int i, ita_nr = 0; /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(the_repository->index); + ensure_full_index_unaudited(the_repository->index); for (i = 0; i < the_repository->index->cache_nr; i++) if (ce_intent_to_add(the_repository->index->cache[i])) ita_nr++; @@ -1221,6 +1338,8 @@ static enum untracked_status_type parse_untracked_setting_name(const char *u) return SHOW_NORMAL_UNTRACKED_FILES; else if (!strcmp(u, "all")) return SHOW_ALL_UNTRACKED_FILES; + else if (!strcmp(u,"complete")) + return SHOW_COMPLETE_UNTRACKED_FILES; else return SHOW_UNTRACKED_FILES_ERROR; } @@ -1531,6 +1650,28 @@ static int git_status_config(const char *k, const char *v, s->relative_paths = git_config_bool(k, v); return 0; } + if (!strcmp(k, "status.deserializepath")) { + /* + * Automatically assume deserialization if this is + * set in the config and the file exists. Do not + * complain if the file does not exist, because we + * silently fall back to normal mode. + */ + if (v && *v && access(v, R_OK) == 0) { + do_implicit_deserialize = 1; + deserialize_path = xstrdup(v); + } else { + reject_implicit = 1; + } + return 0; + } + if (!strcmp(k, "status.deserializewait")) { + if (!v || !*v) + implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + else + implicit_deserialize_wait = parse_dw(v); + return 0; + } if (!strcmp(k, "status.showuntrackedfiles")) { enum untracked_status_type u; @@ -1558,6 +1699,26 @@ static int git_status_config(const char *k, const char *v, s->detect_rename = git_config_rename(k, v); return 0; } + if (!strcmp(k, "diff.renamethreshold")) { + if (s->rename_score == -1) { + const char *arg = v; + if (!v) + return config_error_nonbool(k); + s->rename_score = parse_rename_score(&arg); + if (*arg) + return error(_("invalid value for '%s': '%s'"), k, v); + } + return 0; + } + if (!strcmp(k, "status.renamethreshold")) { + const char *arg = v; + if (!v) + return config_error_nonbool(k); + s->rename_score = parse_rename_score(&arg); + if (*arg) + return error(_("invalid value for '%s': '%s'"), k, v); + return 0; + } return git_diff_ui_config(k, v, ctx, NULL); } @@ -1570,7 +1731,8 @@ struct repository *repo UNUSED) static const char *rename_score_arg = (const char *)-1; static struct wt_status s; unsigned int progress_flag = 0; - int fd; + int try_deserialize; + int fd = -1; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1585,6 +1747,15 @@ struct repository *repo UNUSED) OPT_CALLBACK_F(0, "porcelain", &status_format, N_("version"), N_("machine-readable output"), PARSE_OPT_OPTARG, opt_parse_porcelain), + OPT_CALLBACK_F(0, "serialize", &status_format, + N_("path"), N_("serialize raw status data to path or stdout"), + PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize), + OPT_CALLBACK_F(0, "deserialize", NULL, + N_("path"), N_("deserialize raw status data from file"), + PARSE_OPT_OPTARG, opt_parse_deserialize), + OPT_CALLBACK_F(0, "deserialize-wait", NULL, + N_("fail|block|no"), N_("how to wait if status cache file is invalid"), + PARSE_OPT_OPTARG, opt_parse_deserialize_wait), OPT_SET_INT(0, "long", &status_format, N_("show status in long format (default)"), STATUS_FORMAT_LONG), @@ -1646,10 +1817,53 @@ struct repository *repo UNUSED) s.show_untracked_files == SHOW_NO_UNTRACKED_FILES) die(_("Unsupported combination of ignored and untracked-files arguments")); + if (s.show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES && + s.show_ignored_mode == SHOW_NO_IGNORED) + die(_("Complete Untracked only supported with ignored files")); + parse_pathspec(&s.pathspec, 0, PATHSPEC_PREFER_FULL, prefix, argv); + /* + * If we want to try to deserialize status data from a cache file, + * we need to re-order the initialization code. The problem is that + * this makes for a very nasty diff and causes merge conflicts as we + * carry it forward. And it easy to mess up the merge, so we + * duplicate some code here to hopefully reduce conflicts. + */ + try_deserialize = (!do_serialize && + (do_implicit_deserialize || do_explicit_deserialize)); + + /* + * Disable deserialize when verbose is set because it causes us to + * print diffs for each modified file, but that requires us to have + * the index loaded and we don't want to do that (at least not now for + * this seldom used feature). My fear is that would further tangle + * the merge conflict with upstream. + * + * TODO Reconsider this in the future. + */ + if (try_deserialize && verbose) { + trace2_data_string("status", the_repository, "deserialize/reject", + "args/verbose"); + try_deserialize = 0; + } + + if (try_deserialize) + goto skip_init; + /* + * If we implicitly received a status cache pathname from the config + * and the file does not exist, we silently reject it and do the normal + * status "collect". Fake up some trace2 messages to reflect this and + * assist post-processors know this case is different. + */ + if (!do_serialize && reject_implicit) { + trace2_cmd_mode("implicit-deserialize"); + trace2_data_string("status", the_repository, "deserialize/reject", + "status-cache/access"); + } + enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) @@ -1664,6 +1878,7 @@ struct repository *repo UNUSED) else fd = -1; +skip_init: s.is_initial = repo_get_oid(the_repository, s.reference, &oid) ? 1 : 0; if (!s.is_initial) oidcpy(&s.oid_commit, &oid); @@ -1680,6 +1895,36 @@ struct repository *repo UNUSED) s.rename_score = parse_rename_score(&rename_score_arg); } + if (try_deserialize) { + int result; + enum wt_status_deserialize_wait dw = implicit_deserialize_wait; + if (explicit_deserialize_wait != DESERIALIZE_WAIT__UNSET) + dw = explicit_deserialize_wait; + if (dw == DESERIALIZE_WAIT__UNSET) + dw = DESERIALIZE_WAIT__NO; + + if (s.relative_paths) + s.prefix = prefix; + + trace2_cmd_mode("deserialize"); + result = wt_status_deserialize(&s, deserialize_path, dw); + if (result == DESERIALIZE_OK) + return 0; + if (dw == DESERIALIZE_WAIT__FAIL) + die(_("Rejected status serialization cache")); + + /* deserialize failed, so force the initialization we skipped above. */ + enable_fscache(1); + repo_read_index_preload(the_repository, &s.pathspec, 0); + refresh_index(the_repository->index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); + + if (use_optional_locks()) + fd = repo_hold_locked_index(the_repository, &index_lock, 0); + else + fd = -1; + } + + trace2_cmd_mode("collect"); wt_status_collect(&s); if (0 <= fd) @@ -1688,6 +1933,17 @@ struct repository *repo UNUSED) if (s.relative_paths) s.prefix = prefix; + if (serialize_path) { + int fd_serialize = xopen(serialize_path, + O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd_serialize < 0) + die_errno(_("could not serialize to '%s'"), + serialize_path); + trace2_cmd_mode("serialize"); + wt_status_serialize_v1(fd_serialize, &s); + close(fd_serialize); + } + wt_status_print(&s); wt_status_collect_free_buffers(&s); @@ -1830,6 +2086,7 @@ int cmd_commit(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; @@ -1907,7 +2164,7 @@ int cmd_commit(int argc, if (!stat(git_path_merge_mode(the_repository), &statbuf)) { if (strbuf_read_file(&sb, git_path_merge_mode(the_repository), 0) < 0) die_errno(_("could not read MERGE_MODE")); - if (!strcmp(sb.buf, "no-ff")) + if (!strcmp(sb.buf, "no-ff")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand allow_fast_forward = 0; } if (allow_fast_forward) diff --git a/builtin/difftool.c b/builtin/difftool.c index ba33402308577b..8930e11cc89878 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -607,7 +607,7 @@ static int run_dir_diff(struct repository *repo, ret = run_command(&cmd); /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(&wtindex); + ensure_full_index_unaudited(&wtindex); /* * If the diff includes working copy files and those diff --git a/builtin/fetch.c b/builtin/fetch.c index ab7d58fe02c755..1e0854c2dbe549 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -21,6 +21,9 @@ #include "string-list.h" #include "remote.h" #include "transport.h" +#include "gvfs.h" +#include "gvfs-helper-client.h" +#include "packfile.h" #include "run-command.h" #include "parse-options.h" #include "sigchain.h" @@ -568,7 +571,7 @@ static struct ref *get_ref_map(struct remote *remote, if (remote && (remote->fetch.nr || /* Note: has_merge implies non-NULL branch->remote_name */ - (has_merge && !strcmp(branch->remote_name, remote->name)))) { + (has_merge && branch && !strcmp(branch->remote_name, remote->name)))) { for (i = 0; i < remote->fetch.nr; i++) { get_fetch_map(remote_refs, &remote->fetch.items[i], &tail, 0); if (remote->fetch.items[i].dst && @@ -586,6 +589,7 @@ static struct ref *get_ref_map(struct remote *remote, * Note: has_merge implies non-NULL branch->remote_name */ if (has_merge && + branch && !strcmp(branch->remote_name, remote->name)) add_merge_config(&ref_map, remote_refs, branch, &tail); } else if (!prefetch) { @@ -1250,6 +1254,13 @@ static int store_updated_refs(struct display_state *display_state, opt.exclude_hidden_refs_section = "fetch"; rm = ref_map; + + /* + * Before checking connectivity, be really sure we have the + * latest pack-files loaded into memory. + */ + odb_reprepare(the_repository->objects); + if (check_connected(iterate_ref_map, &rm, &opt)) { rc = error(_("%s did not send all necessary objects"), display_state->url); @@ -2506,7 +2517,7 @@ static int fetch_one(struct remote *remote, int argc, const char **argv, int cmd_fetch(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct fetch_config config = { .display_format = DISPLAY_FORMAT_FULL, @@ -2783,6 +2794,9 @@ int cmd_fetch(int argc, } string_list_remove_duplicates(&list, 0); + if (gvfs_config_is_set(repo, GVFS_PREFETCH_DURING_FETCH)) + gh_client__prefetch(0, NULL); + if (negotiate_only) { struct oidset acked_commits = OIDSET_INIT; struct oidset_iter iter; diff --git a/builtin/fsck.c b/builtin/fsck.c index 9af4cc085bc3c1..4a2da5f34a20d8 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -766,7 +766,7 @@ static void fsck_index(struct index_state *istate, const char *index_path, unsigned int i; /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(istate); + ensure_full_index_unaudited(istate); for (i = 0; i < istate->cache_nr; i++) { unsigned int mode; struct blob *blob; diff --git a/builtin/gc.c b/builtin/gc.c index 57a3520263d7be..99d6424eff54ef 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -13,12 +13,15 @@ #define USE_THE_REPOSITORY_VARIABLE #define DISABLE_SIGN_COMPARE_WARNINGS +#include "git-compat-util.h" #include "builtin.h" #include "abspath.h" +#include "copy.h" #include "date.h" #include "dir.h" #include "environment.h" #include "hex.h" +#include "gvfs.h" #include "config.h" #include "tempfile.h" #include "lockfile.h" @@ -222,6 +225,7 @@ enum maintenance_task_label { TASK_REFLOG_EXPIRE, TASK_WORKTREE_PRUNE, TASK_RERERE_GC, + TASK_CACHE_LOCAL_OBJS, /* Leave as final value */ TASK__COUNT @@ -566,7 +570,7 @@ static int maintenance_task_odb(struct maintenance_run_opts *opts, int cmd_gc(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int aggressive = 0; int force = 0; @@ -635,6 +639,14 @@ int cmd_gc(int argc, if (cfg.prune_expire && parse_expiry_date(cfg.prune_expire, &dummy)) die(_("failed to parse prune expiry value %s"), cfg.prune_expire); + if (gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) { + int gc_auto_threshold = 6700; + if (!opts.auto_flag || + repo_config_get_int(repo, "gc.auto", &gc_auto_threshold) || + gc_auto_threshold > 0) + die(_("'git gc' is not supported on a GVFS repo")); + } + if (opts.auto_flag) { struct odb_optimize_options optimize_opts = { .strategy = ODB_OPTIMIZE_INCREMENTAL, @@ -1021,12 +1033,23 @@ static int write_loose_object_to_stdin(const struct object_id *oid, return ++(d->count) > d->batch_size; } +static const char *shared_object_dir = NULL; + static int pack_loose(struct maintenance_run_opts *opts) { struct repository *r = the_repository; int result = 0; struct write_loose_object_data data; struct child_process pack_proc = CHILD_PROCESS_INIT; + struct odb_source *prev_source = NULL; + const char *object_dir = r->objects->sources->path; + + /* If set, use the shared object directory. */ + if (shared_object_dir) { + odb_set_temporary_primary_source(r->objects, shared_object_dir, + 0, &prev_source); + object_dir = shared_object_dir; + } /* * Do not start pack-objects process @@ -1034,8 +1057,12 @@ static int pack_loose(struct maintenance_run_opts *opts) */ if (!for_each_loose_file_in_source(r->objects->sources, bail_on_loose, - NULL, NULL, NULL)) + NULL, NULL, NULL)) { + if (shared_object_dir) + odb_restore_primary_source(r->objects, prev_source, + shared_object_dir); return 0; + } pack_proc.git_cmd = 1; @@ -1044,7 +1071,7 @@ static int pack_loose(struct maintenance_run_opts *opts) strvec_push(&pack_proc.args, "--quiet"); else strvec_push(&pack_proc.args, "--no-quiet"); - strvec_pushf(&pack_proc.args, "%s/pack/loose", r->objects->sources->path); + strvec_pushf(&pack_proc.args, "%s/pack/loose", object_dir); pack_proc.in = -1; @@ -1056,6 +1083,9 @@ static int pack_loose(struct maintenance_run_opts *opts) if (start_command(&pack_proc)) { error(_("failed to start 'git pack-objects' process")); + if (shared_object_dir) + odb_restore_primary_source(r->objects, prev_source, + shared_object_dir); return 1; } @@ -1083,6 +1113,10 @@ static int pack_loose(struct maintenance_run_opts *opts) result = 1; } + if (shared_object_dir) + odb_restore_primary_source(r->objects, prev_source, + shared_object_dir); + return result; } @@ -1260,6 +1294,188 @@ static int geometric_repack_auto_condition(struct gc_config *cfg) return odb_optimize_required(the_repository->objects, &opts); } +static void link_or_copy_or_die(const char *src, const char *dst) +{ + if (!link(src, dst)) + return; + + /* Use copy operation if src and dst are on different file systems. */ + if (errno != EXDEV) + warning_errno(_("failed to link '%s' to '%s'"), src, dst); + + if (copy_file(the_repository, dst, src, 0444)) + die_errno(_("failed to copy '%s' to '%s'"), src, dst); +} + +static void rename_or_copy_or_die(const char *src, const char *dst) +{ + if (!rename(src, dst)) + return; + + /* Use copy and delete if src and dst are on different file systems. */ + if (errno != EXDEV) + warning_errno(_("failed to move '%s' to '%s'"), src, dst); + + if (copy_file(the_repository, dst, src, 0444)) + die_errno(_("failed to copy '%s' to '%s'"), src, dst); + + if (unlink(src)) + die_errno(_("failed to delete '%s'"), src); +} + +static void migrate_pack(const char *srcdir, const char *dstdir, + const char *pack_filename) +{ + size_t basenamelen, srclen, dstlen; + struct strbuf src = STRBUF_INIT, dst = STRBUF_INIT; + struct { + const char *ext; + unsigned move:1; + } files[] = { + {".pack", 0}, + {".keep", 0}, + {".rev", 0}, + {".idx", 1}, /* The index file must be atomically moved last. */ + }; + + trace2_region_enter("maintenance", "migrate_pack", the_repository); + + basenamelen = strlen(pack_filename) - 5; /* .pack */ + strbuf_addstr(&src, srcdir); + strbuf_addch(&src, '/'); + strbuf_add(&src, pack_filename, basenamelen); + strbuf_addstr(&src, ".idx"); + + /* A pack without an index file is not yet ready to be migrated. */ + if (!file_exists(src.buf)) + goto cleanup; + + strbuf_setlen(&src, src.len - 4 /* .idx */); + strbuf_addstr(&dst, dstdir); + strbuf_addch(&dst, '/'); + strbuf_add(&dst, pack_filename, basenamelen); + + srclen = src.len; + dstlen = dst.len; + + /* Move or copy files from the source directory to the destination. */ + for (size_t i = 0; i < ARRAY_SIZE(files); i++) { + strbuf_setlen(&src, srclen); + strbuf_addstr(&src, files[i].ext); + + if (!file_exists(src.buf)) + continue; + + strbuf_setlen(&dst, dstlen); + strbuf_addstr(&dst, files[i].ext); + + if (files[i].move) + rename_or_copy_or_die(src.buf, dst.buf); + else + link_or_copy_or_die(src.buf, dst.buf); + } + + /* + * Now the pack and all associated files exist at the destination we can + * now clean up the files in the source directory. + */ + for (size_t i = 0; i < ARRAY_SIZE(files); i++) { + /* Files that were moved rather than copied have no clean up. */ + if (files[i].move) + continue; + + strbuf_setlen(&src, srclen); + strbuf_addstr(&src, files[i].ext); + + /* Files that never existed in originally have no clean up.*/ + if (!file_exists(src.buf)) + continue; + + if (unlink(src.buf)) + warning_errno(_("failed to delete '%s'"), src.buf); + } + +cleanup: + strbuf_release(&src); + strbuf_release(&dst); + + trace2_region_leave("maintenance", "migrate_pack", the_repository); +} + +static void move_pack_to_shared_cache(const char *full_path, size_t full_path_len, + const char *file_name, void *data) +{ + char *srcdir; + const char *dstdir = (const char *)data; + + /* We only care about the actual pack files here. + * The associated .idx, .keep, .rev files will be copied in tandem + * with the pack file, with the index file being moved last. + * The original locations of the non-index files will only deleted + * once all other files have been copied/moved. + */ + if (!ends_with(file_name, ".pack")) + return; + + srcdir = xstrndup(full_path, full_path_len - strlen(file_name) - 1); + + migrate_pack(srcdir, dstdir, file_name); + + free(srcdir); +} + +static int move_loose_object_to_shared_cache(const struct object_id *oid, + const char *path, + UNUSED void *data) +{ + struct stat st; + struct strbuf dst = STRBUF_INIT; + char *hex = oid_to_hex(oid); + + strbuf_addf(&dst, "%s/%.2s/", shared_object_dir, hex); + + if (stat(dst.buf, &st)) { + if (mkdir(dst.buf, 0777)) + die_errno(_("failed to create directory '%s'"), dst.buf); + } else if (!S_ISDIR(st.st_mode)) + die(_("expected '%s' to be a directory"), dst.buf); + + strbuf_addstr(&dst, hex+2); + rename_or_copy_or_die(path, dst.buf); + + strbuf_release(&dst); + return 0; +} + +static int maintenance_task_cache_local_objs(UNUSED struct maintenance_run_opts *opts, + UNUSED struct gc_config *cfg) +{ + struct strbuf dstdir = STRBUF_INIT; + struct repository *r = the_repository; + int ret = 0; + + /* This task is only applicable with a VFS/Scalar shared cache. */ + if (!shared_object_dir) + return 0; + + /* If the dest is the same as the local odb path then we do nothing. */ + if (!fspathcmp(r->objects->sources->path, shared_object_dir)) + goto cleanup; + + strbuf_addf(&dstdir, "%s/pack", shared_object_dir); + + for_each_file_in_pack_dir(r->objects->sources->path, move_pack_to_shared_cache, + dstdir.buf); + + ret = for_each_loose_file_in_source(r->objects->sources, + move_loose_object_to_shared_cache, + NULL, NULL, NULL); + +cleanup: + strbuf_release(&dstdir); + return ret; +} + typedef int (*maintenance_task_fn)(struct maintenance_run_opts *opts, struct gc_config *cfg); typedef int (*maintenance_auto_fn)(struct gc_config *cfg); @@ -1338,6 +1554,10 @@ static const struct maintenance_task tasks[] = { .background = maintenance_task_rerere_gc, .auto_condition = rerere_gc_condition, }, + [TASK_CACHE_LOCAL_OBJS] = { + "cache-local-objects", + maintenance_task_cache_local_objs, + }, }; enum task_phase { @@ -1482,6 +1702,10 @@ static const struct maintenance_strategy incremental_strategy = { .type = MAINTENANCE_TYPE_SCHEDULED, .schedule = SCHEDULE_WEEKLY, }, + [TASK_CACHE_LOCAL_OBJS] = { + .type = MAINTENANCE_TYPE_SCHEDULED, + .schedule = SCHEDULE_WEEKLY, + }, /* * Historically, the "incremental" strategy was only available * in the context of scheduled maintenance when set up via @@ -1641,11 +1865,12 @@ static int task_option_parse(const struct option *opt, } static int maintenance_run(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct maintenance_run_opts opts = MAINTENANCE_RUN_OPTS_INIT; struct string_list selected_tasks = STRING_LIST_INIT_DUP; struct gc_config cfg = GC_CONFIG_INIT; + const char *tmp_obj_dir = NULL; struct option builtin_maintenance_run_options[] = { OPT_BOOL(0, "auto", &opts.auto_flag, N_("run tasks based on the state of the repository")), @@ -1682,6 +1907,17 @@ static int maintenance_run(int argc, const char **argv, const char *prefix, usage_with_options(builtin_maintenance_run_usage, builtin_maintenance_run_options); + /* + * To enable the VFS for Git/Scalar shared object cache, use + * the gvfs.sharedcache config option to redirect the + * maintenance to that location. + */ + if (!repo_config_get_value(repo, "gvfs.sharedcache", &tmp_obj_dir) && + tmp_obj_dir) { + shared_object_dir = xstrdup(tmp_obj_dir); + setenv(DB_ENVIRONMENT, shared_object_dir, 1); + } + ret = maintenance_run_tasks(&opts, &cfg); string_list_clear(&selected_tasks, 0); diff --git a/builtin/help.c b/builtin/help.c index a140339999debe..b99b0bc1040d82 100644 --- a/builtin/help.c +++ b/builtin/help.c @@ -302,7 +302,7 @@ static void exec_woman_emacs(const char *path, const char *page) if (!path) path = "emacsclient"; strbuf_addf(&man_page, "(woman \"%s\")", page); - execlp(path, "emacsclient", "-e", man_page.buf, (char *)NULL); + execlp(path, "emacsclient", "-e", man_page.buf, (char *)NULL); // CodeQL [SM01925] justification: Git's help system safely consumes user-controlled environment variables and paths warning_errno(_("failed to exec '%s'"), path); strbuf_release(&man_page); } @@ -324,7 +324,7 @@ static void exec_man_konqueror(const char *path, const char *page) } else path = "kfmclient"; strbuf_addf(&man_page, "man:%s(1)", page); - execlp(path, filename, "newTab", man_page.buf, (char *)NULL); + execlp(path, filename, "newTab", man_page.buf, (char *)NULL); // CodeQL [SM01925] justification: Git's help system safely consumes user-controlled environment variables and paths warning_errno(_("failed to exec '%s'"), path); strbuf_release(&man_page); } @@ -334,7 +334,7 @@ static void exec_man_man(const char *path, const char *page) { if (!path) path = "man"; - execlp(path, "man", page, (char *)NULL); + execlp(path, "man", page, (char *)NULL); // CodeQL [SM01925] justification: Git's help system safely consumes user-controlled environment variables and paths warning_errno(_("failed to exec '%s'"), path); } diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 6b2a87e2d39355..5028a92f069877 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -892,7 +892,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry, if (startup_info->have_repository) { read_lock(); collision_test_needed = odb_has_object(the_repository->objects, oid, - ODB_HAS_OBJECT_FETCH_PROMISOR); + 0); read_unlock(); } @@ -1902,6 +1902,7 @@ int cmd_index_pack(int argc, unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */ int report_end_of_input = 0; int hash_algo = 0; + int dash_o = 0; show_usage_if_asked(argc, argv, index_pack_usage); @@ -1988,6 +1989,7 @@ int cmd_index_pack(int argc, if (index_name || (i+1) >= argc) usage(index_pack_usage); index_name = argv[++i]; + dash_o = 1; } else if (starts_with(arg, "--index-version=")) { char *c; opts.version = strtoul(arg + 16, &c, 10); @@ -2041,6 +2043,8 @@ int cmd_index_pack(int argc, repo_set_hash_algo(the_repository, GIT_HASH_DEFAULT); opts.flags &= ~(WRITE_REV | WRITE_REV_VERIFY); + if (rev_index && dash_o && !ends_with(index_name, ".idx")) + rev_index = 0; if (rev_index) { opts.flags |= verify ? WRITE_REV_VERIFY : WRITE_REV; if (index_name) diff --git a/builtin/ls-files.c b/builtin/ls-files.c index b044520f9e3c39..c37a7b4def0b5f 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -428,7 +428,7 @@ static void show_files(struct repository *repo, struct dir_struct *dir) * so expansion will leave the first 'i' entries * alone. */ - ensure_full_index(repo->index); + ensure_full_index_with_reason(repo->index, "ls-files"); ce = repo->index->cache[i]; } diff --git a/builtin/merge-index.c b/builtin/merge-index.c index 3314fb13361d64..b0bc54d2e473c3 100644 --- a/builtin/merge-index.c +++ b/builtin/merge-index.c @@ -66,7 +66,7 @@ static void merge_all(void) { int i; /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(the_repository->index); + ensure_full_index_unaudited(the_repository->index); for (i = 0; i < the_repository->index->cache_nr; i++) { const struct cache_entry *ce = the_repository->index->cache[i]; if (!ce_stage(ce)) @@ -98,7 +98,7 @@ int cmd_merge_index(int argc, repo_read_index(the_repository); /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(the_repository->index); + ensure_full_index_unaudited(the_repository->index); i = 1; if (!strcmp(argv[i], "-o")) { diff --git a/builtin/merge.c b/builtin/merge.c index 5b4eb23a833295..02a87430f48f97 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -1382,6 +1382,7 @@ int cmd_merge(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/push.c b/builtin/push.c index 2377b5af554bda..542eb91f404a9e 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -93,7 +93,7 @@ static void refspec_append_mapped(struct refspec *refspec, const char *ref, if (cfg->push_default == PUSH_DEFAULT_UPSTREAM && skip_prefix(matched->name, "refs/heads/", &branch_name)) { struct branch *branch = branch_get(branch_name); - if (branch->merge_nr == 1 && branch->merge[0]->src) { + if (branch && branch->merge_nr == 1 && branch->merge[0]->src) { refspec_appendf(refspec, "%s:%s", ref, branch->merge[0]->src); return; @@ -767,6 +767,10 @@ int cmd_push(int argc, else if (recurse_submodules == RECURSE_SUBMODULES_ONLY) flags |= TRANSPORT_RECURSE_SUBMODULES_ONLY; + prepare_repo_settings(the_repository); + if (the_repository->settings.pack_use_path_walk) + flags |= TRANSPORT_PUSH_NO_REUSE_DELTA; + if (argc > 0) repo = argv[0]; diff --git a/builtin/read-tree.c b/builtin/read-tree.c index 999a82ecdfd737..70b4c233adc9d0 100644 --- a/builtin/read-tree.c +++ b/builtin/read-tree.c @@ -232,7 +232,8 @@ int cmd_read_tree(int argc, setup_work_tree(the_repository); if (opts.skip_sparse_checkout) - ensure_full_index(the_repository->index); + ensure_full_index_with_reason(the_repository->index, + "read-tree"); if (opts.merge) { switch (stage - 1) { diff --git a/builtin/rebase.c b/builtin/rebase.c index 10a306310cd439..80daa6b1bbf391 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -489,9 +489,9 @@ static int read_basic_state(struct rebase_options *opts) if (!read_oneliner(&buf, state_dir_path("allow_rerere_autoupdate", opts), READ_ONELINER_WARN_MISSING)) return -1; - if (!strcmp(buf.buf, "--rerere-autoupdate")) + if (!strcmp(buf.buf, "--rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_autoupdate = RERERE_AUTOUPDATE; - else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) + else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_autoupdate = RERERE_NOAUTOUPDATE; else warning(_("ignoring invalid allow_rerere_autoupdate: " @@ -1257,6 +1257,7 @@ int cmd_rebase(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/repack.c b/builtin/repack.c index c4360382c1fce2..a8652e2d6d447e 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -19,6 +19,7 @@ #include "hex.h" #include "wt-status.h" #include "read-cache-ll.h" +#include "gvfs.h" #define ALL_INTO_ONE 1 #define LOOSEN_UNREACHABLE 2 @@ -169,6 +170,7 @@ int cmd_repack(int argc, struct tempfile *refs_snapshot = NULL; int i, ret; int show_progress; + const char *tmp_obj_dir = NULL; /* variables to be filled by option parsing */ struct repack_config_ctx config_ctx; @@ -430,6 +432,10 @@ int cmd_repack(int argc, write_bitmaps = 0; } + if (gvfs_config_is_set(repo, GVFS_ANY_MASK) && + !repo_config_get_value(repo, "gvfs.sharedcache", &tmp_obj_dir)) + warning(_("shared object cache is configured but will not be repacked")); + if (config_ctx.midx_split_factor < 2) die(_("invalid value for %s: %d"), "--midx-split-factor", config_ctx.midx_split_factor); diff --git a/builtin/repo.c b/builtin/repo.c index 1d3a4a79523202..ef87405832353d 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -15,10 +15,13 @@ #include "ref-filter.h" #include "refs.h" #include "revision.h" +#include "run-command.h" #include "setup.h" #include "strbuf.h" #include "string-list.h" +#include "strvec.h" #include "shallow.h" +#include "trace2.h" #include "tree.h" #include "tree-walk.h" #include "utf8.h" @@ -307,6 +310,19 @@ struct object_data { size_t value; }; +struct top_object { + struct object_data object; + char *path; + struct object_id containing_commit_oid; + char *name_rev; +}; + +struct top_objects { + size_t nr; + size_t alloc; + struct top_object *data; +}; + struct largest_objects { struct object_data tag_size; struct object_data commit_size; @@ -357,6 +373,13 @@ struct ref_stats { size_t tags; size_t annotated_tags; size_t others; + size_t symbolic; + size_t loose; + size_t packed; + size_t max_local_refname_length; + size_t total_local_refname_length; + size_t max_remote_refname_length; + size_t total_remote_refname_length; }; struct object_values { @@ -366,6 +389,19 @@ struct object_values { size_t blobs; }; +/* Log16 size buckets and log4 entry buckets cover the full size_t range. */ +#define HBIN_SHIFT 4 +#define HBIN_LEN (sizeof(size_t) * CHAR_BIT / HBIN_SHIFT) +#define QBIN_SHIFT 2 +#define QBIN_LEN (sizeof(size_t) * CHAR_BIT / QBIN_SHIFT) +#define PBIN_VEC_LEN 32 + +struct object_histogram_bin { + size_t count; + size_t inflated_size; + size_t disk_size; +}; + struct object_stats { struct object_values type_counts; struct object_values inflated_sizes; @@ -373,6 +409,16 @@ struct object_stats { struct largest_objects largest; struct top_paths top_trees; struct top_paths top_blobs; + struct object_histogram_bin commit_sizes[HBIN_LEN]; + struct object_histogram_bin tree_sizes[HBIN_LEN]; + struct object_histogram_bin blob_sizes[HBIN_LEN]; + struct object_histogram_bin tree_entries[QBIN_LEN]; + size_t commit_parents[PBIN_VEC_LEN]; + struct top_objects top_commit_parents; + struct top_objects top_commit_sizes; + struct top_objects top_tree_entries; + struct top_objects top_tree_sizes; + struct top_objects top_blob_sizes; }; struct repo_structure { @@ -547,6 +593,21 @@ static void stats_table_setup_structure(struct stats_table *table, " * %s", _("Annotated")); stats_table_count_addf(table, refs->remotes, " * %s", _("Remotes")); stats_table_count_addf(table, refs->others, " * %s", _("Others")); + stats_table_count_addf(table, refs->symbolic, + " * %s", _("Symbolic refs")); + stats_table_count_addf(table, refs->loose, " * %s", _("Loose refs")); + stats_table_count_addf(table, refs->packed, " * %s", _("Packed refs")); + stats_table_addf(table, " * %s", _("Refname length")); + stats_table_addf(table, " * %s", _("Local")); + stats_table_count_addf(table, refs->max_local_refname_length, + " * %s", _("Maximum")); + stats_table_count_addf(table, refs->total_local_refname_length, + " * %s", _("Total")); + stats_table_addf(table, " * %s", _("Remote")); + stats_table_count_addf(table, refs->max_remote_refname_length, + " * %s", _("Maximum")); + stats_table_count_addf(table, refs->total_remote_refname_length, + " * %s", _("Total")); object_count_total = get_total_object_values(&objects->type_counts); stats_table_addf(table, ""); @@ -654,25 +715,29 @@ static void stats_table_setup_top_paths(struct stats_table *table, #define INDEX_WIDTH 4 -static void stats_table_print_structure(const struct stats_table *table) +static void stats_table_print(const struct stats_table *table, + const char *name_col_title) { - const char *name_col_title = _("Repository structure"); const char *value_col_title = _("Value"); int title_name_width = utf8_strwidth(name_col_title); int title_value_width = utf8_strwidth(value_col_title); int name_col_width = table->name_col_width; int value_col_width = table->value_col_width; int unit_col_width = table->unit_col_width; + int index_width = INDEX_WIDTH; struct string_list_item *item; struct strbuf buf = STRBUF_INIT; + for (size_t n = table->annotations.nr; n >= 10; n /= 10) + index_width++; + if (title_name_width > name_col_width) name_col_width = title_name_width; if (title_value_width > value_col_width + unit_col_width + 1) value_col_width = title_value_width - unit_col_width; strbuf_addstr(&buf, "| "); - strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width + INDEX_WIDTH, + strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width + index_width, name_col_title); strbuf_addstr(&buf, " | "); strbuf_utf8_align(&buf, ALIGN_LEFT, @@ -681,7 +746,7 @@ static void stats_table_print_structure(const struct stats_table *table) printf("%s\n", buf.buf); printf("| "); - for (int i = 0; i < name_col_width + INDEX_WIDTH; i++) + for (int i = 0; i < name_col_width + index_width; i++) putchar('-'); printf(" | "); for (int i = 0; i < value_col_width + unit_col_width + 1; i++) @@ -703,11 +768,16 @@ static void stats_table_print_structure(const struct stats_table *table) strbuf_addstr(&buf, "| "); strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width, item->string); - if (entry && entry->oid) + if (entry && entry->oid) { + size_t len = buf.len; + strbuf_addf(&buf, " [%" PRIuMAX "]", (uintmax_t)entry->index); - else - strbuf_addchars(&buf, ' ', INDEX_WIDTH); + strbuf_addchars(&buf, ' ', + index_width - (buf.len - len)); + } else { + strbuf_addchars(&buf, ' ', index_width); + } strbuf_addstr(&buf, " | "); strbuf_utf8_align(&buf, ALIGN_RIGHT, value_col_width, value); @@ -741,6 +811,135 @@ static void stats_table_clear(struct stats_table *table) string_list_clear(&table->annotations, 1); } +static void histogram_table_print(const char *title, + const struct object_histogram_bin *bins, + size_t nr, unsigned int shift) +{ + struct stats_table table = { + .rows = STRING_LIST_INIT_DUP, + .annotations = STRING_LIST_INIT_DUP, + }; + + for (size_t i = 0; i < nr; i++) { + size_t lower, upper; + + if (!bins[i].count) + continue; + + lower = i ? (size_t)1 << (i * shift) : 0; + upper = SIZE_MAX >> (sizeof(size_t) * CHAR_BIT - + (i + 1) * shift); + stats_table_addf(&table, "* %" PRIuMAX "..%" PRIuMAX, + (uintmax_t)lower, (uintmax_t)upper); + stats_table_count_addf(&table, bins[i].count, + " * %s", _("Count")); + stats_table_size_addf(&table, bins[i].inflated_size, + " * %s", _("Inflated size")); + stats_table_size_addf(&table, bins[i].disk_size, + " * %s", _("Disk size")); + } + + if (table.rows.nr) { + putchar('\n'); + stats_table_print(&table, title); + } + stats_table_clear(&table); +} + +static void structure_histograms_table_print(struct object_stats *stats) +{ + struct stats_table table = { + .rows = STRING_LIST_INIT_DUP, + .annotations = STRING_LIST_INIT_DUP, + }; + + for (size_t i = 0; i < ARRAY_SIZE(stats->commit_parents); i++) { + if (!stats->commit_parents[i]) + continue; + stats_table_count_addf(&table, stats->commit_parents[i], + "%" PRIuMAX "%s", (uintmax_t)i, + i == PBIN_VEC_LEN - 1 ? "+" : ""); + } + + if (table.rows.nr) { + putchar('\n'); + stats_table_print(&table, _("Commit parent histogram")); + } + stats_table_clear(&table); + + histogram_table_print(_("Commit size histogram"), stats->commit_sizes, + ARRAY_SIZE(stats->commit_sizes), HBIN_SHIFT); + histogram_table_print(_("Tree entry histogram"), stats->tree_entries, + ARRAY_SIZE(stats->tree_entries), QBIN_SHIFT); + histogram_table_print(_("Tree size histogram"), stats->tree_sizes, + ARRAY_SIZE(stats->tree_sizes), HBIN_SHIFT); + histogram_table_print(_("Blob size histogram"), stats->blob_sizes, + ARRAY_SIZE(stats->blob_sizes), HBIN_SHIFT); +} + +static void top_objects_table_print(const char *title, struct top_objects *top, + int by_size) +{ + struct stats_table table = { + .rows = STRING_LIST_INIT_DUP, + .annotations = STRING_LIST_INIT_DUP, + }; + struct strbuf label = STRBUF_INIT; + + for (size_t i = 0; i < top->nr; i++) { + struct object_data *item = &top->data[i].object; + const char *path = top->data[i].path; + const char *name_rev = top->data[i].name_rev; + const struct object_id *commit_oid = + &top->data[i].containing_commit_oid; + + strbuf_reset(&label); + strbuf_addf(&label, "%" PRIuMAX, (uintmax_t)(i + 1)); + if (path && *path) { + strbuf_addstr(&label, ": "); + quote_c_style(path, &label, NULL, 0); + } + if (!is_null_oid(commit_oid)) + strbuf_addf(&label, _(" (commit %s)"), + oid_to_hex(commit_oid)); + if (name_rev) { + strbuf_addstr(&label, " ("); + quote_c_style(name_rev, &label, NULL, 0); + strbuf_addch(&label, ')'); + } + + if (by_size) + stats_table_object_size_addf(&table, &item->oid, + item->value, + "%s", label.buf); + else + stats_table_object_count_addf(&table, &item->oid, + item->value, + "%s", label.buf); + } + + if (table.rows.nr) { + putchar('\n'); + stats_table_print(&table, title); + } + strbuf_release(&label); + stats_table_clear(&table); +} + +static void structure_top_objects_table_print(struct object_stats *stats) +{ + top_objects_table_print(_("Largest commits by parent count"), + &stats->top_commit_parents, 0); + top_objects_table_print(_("Largest commits by size"), + &stats->top_commit_sizes, 1); + top_objects_table_print(_("Largest trees by entry count"), + &stats->top_tree_entries, 0); + top_objects_table_print(_("Largest trees by size"), + &stats->top_tree_sizes, 1); + top_objects_table_print(_("Largest blobs by size"), + &stats->top_blob_sizes, 1); +} + static inline void print_keyvalue(const char *key, char key_delim, size_t value, char value_delim) { @@ -804,6 +1003,57 @@ static void top_paths_keyvalue_print(const char *prefix, } } +static void histogram_keyvalue_print(const char *prefix, + const struct object_histogram_bin *bins, + size_t nr, char key_delim, + char value_delim) +{ + for (size_t i = 0; i < nr; i++) { + if (!bins[i].count) + continue; + + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)i); + print_keyvalue("count", key_delim, bins[i].count, value_delim); + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)i); + print_keyvalue("inflated_size", key_delim, + bins[i].inflated_size, value_delim); + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)i); + print_keyvalue("disk_size", key_delim, + bins[i].disk_size, value_delim); + } +} + +static void top_objects_keyvalue_print(const char *prefix, const char *metric, + const struct top_objects *top, + char key_delim, char value_delim) +{ + for (size_t i = 0; i < top->nr; i++) { + const struct top_object *item = &top->data[i]; + + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)(i + 1)); + print_keyvalue(metric, key_delim, item->object.value, + value_delim); + printf("%s.%" PRIuMAX ".oid%c%s%c", prefix, + (uintmax_t)(i + 1), key_delim, + oid_to_hex(&item->object.oid), value_delim); + if (item->path) { + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)(i + 1)); + print_keyvalue_path("path", key_delim, item->path, + value_delim); + } + if (!is_null_oid(&item->containing_commit_oid)) + printf("%s.%" PRIuMAX ".commit_oid%c%s%c", + prefix, (uintmax_t)(i + 1), key_delim, + oid_to_hex(&item->containing_commit_oid), + value_delim); + if (item->name_rev) { + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)(i + 1)); + print_keyvalue_path("name_rev", key_delim, + item->name_rev, value_delim); + } + } +} + static void structure_keyvalue_print(struct repo_structure *stats, char key_delim, char value_delim) { @@ -817,6 +1067,20 @@ static void structure_keyvalue_print(struct repo_structure *stats, stats->refs.remotes, value_delim); print_keyvalue("references.others.count", key_delim, stats->refs.others, value_delim); + print_keyvalue("references.symbolic.count", key_delim, + stats->refs.symbolic, value_delim); + print_keyvalue("references.loose.count", key_delim, + stats->refs.loose, value_delim); + print_keyvalue("references.packed.count", key_delim, + stats->refs.packed, value_delim); + print_keyvalue("references.local.max_length", key_delim, + stats->refs.max_local_refname_length, value_delim); + print_keyvalue("references.local.total_length", key_delim, + stats->refs.total_local_refname_length, value_delim); + print_keyvalue("references.remotes.max_length", key_delim, + stats->refs.max_remote_refname_length, value_delim); + print_keyvalue("references.remotes.total_length", key_delim, + stats->refs.total_remote_refname_length, value_delim); print_keyvalue("objects.commits.count", key_delim, stats->objects.type_counts.commits, value_delim); @@ -864,6 +1128,52 @@ static void structure_keyvalue_print(struct repo_structure *stats, top_paths_keyvalue_print("objects.blobs.top", &stats->objects.top_blobs, key_delim, value_delim); + for (size_t i = 0; i < ARRAY_SIZE(stats->objects.commit_parents); i++) { + if (!stats->objects.commit_parents[i]) + continue; + printf("objects.commits.histogram.parents.%" PRIuMAX ".", + (uintmax_t)i); + print_keyvalue("count", key_delim, + stats->objects.commit_parents[i], value_delim); + } + + histogram_keyvalue_print("objects.commits.histogram.size", + stats->objects.commit_sizes, + ARRAY_SIZE(stats->objects.commit_sizes), + key_delim, value_delim); + histogram_keyvalue_print("objects.trees.histogram.entries", + stats->objects.tree_entries, + ARRAY_SIZE(stats->objects.tree_entries), + key_delim, value_delim); + histogram_keyvalue_print("objects.trees.histogram.size", + stats->objects.tree_sizes, + ARRAY_SIZE(stats->objects.tree_sizes), + key_delim, value_delim); + histogram_keyvalue_print("objects.blobs.histogram.size", + stats->objects.blob_sizes, + ARRAY_SIZE(stats->objects.blob_sizes), + key_delim, value_delim); + + top_objects_keyvalue_print("objects.commits.largest.by_parents", + "parents", + &stats->objects.top_commit_parents, + key_delim, value_delim); + top_objects_keyvalue_print("objects.commits.largest.by_size", + "inflated_size", + &stats->objects.top_commit_sizes, + key_delim, value_delim); + top_objects_keyvalue_print("objects.trees.largest.by_entries", + "entries", &stats->objects.top_tree_entries, + key_delim, value_delim); + top_objects_keyvalue_print("objects.trees.largest.by_size", + "inflated_size", + &stats->objects.top_tree_sizes, + key_delim, value_delim); + top_objects_keyvalue_print("objects.blobs.largest.by_size", + "inflated_size", + &stats->objects.top_blob_sizes, + key_delim, value_delim); + fflush(stdout); } @@ -898,12 +1208,14 @@ static int count_references(const struct reference *ref, void *cb_data) { struct count_references_data *data = cb_data; struct ref_stats *stats = data->stats; - size_t ref_count; + size_t ref_count, refname_length; + unsigned int ref_kind; if (!ref_matches_any_filter(ref->name, data->filters)) return 0; - switch (ref_kind_from_refname(ref->name)) { + ref_kind = ref_kind_from_refname(ref->name); + switch (ref_kind) { case FILTER_REFS_BRANCHES: stats->branches++; break; @@ -923,6 +1235,29 @@ static int count_references(const struct reference *ref, void *cb_data) BUG("unexpected reference type"); } + if (ref->flags & REF_ISSYMREF) + stats->symbolic++; + + if (data->repo->ref_storage_format == REF_STORAGE_FORMAT_FILES) { + /* A symref can inherit REF_ISPACKED from its target. */ + if ((ref->flags & REF_ISPACKED) && + !(ref->flags & REF_ISSYMREF)) + stats->packed++; + else + stats->loose++; + } + + refname_length = strlen(ref->name); + if (ref_kind == FILTER_REFS_REMOTES) { + stats->total_remote_refname_length += refname_length; + if (refname_length > stats->max_remote_refname_length) + stats->max_remote_refname_length = refname_length; + } else { + stats->total_local_refname_length += refname_length; + if (refname_length > stats->max_local_refname_length) + stats->max_local_refname_length = refname_length; + } + /* * While iterating through references for counting, also add OIDs in * preparation for the path walk. @@ -1051,6 +1386,52 @@ static void check_largest(struct object_data *data, struct object_id *oid, } } +static void init_top_objects(struct top_objects *top, int limit, + const char *option) +{ + if (limit < 0) + die(_("--%s= must be non-negative"), option); + top->alloc = limit; + if (limit) + ALLOC_ARRAY(top->data, top->alloc); +} + +static void clear_top_objects(struct top_objects *top) +{ + for (size_t i = 0; i < top->nr; i++) { + free(top->data[i].path); + free(top->data[i].name_rev); + } + free(top->data); +} + +static void maybe_insert_top_object(struct top_objects *top, + const struct object_id *oid, size_t value, + const char *path, + const struct object_id *commit_oid) +{ + size_t pos = top->nr; + + while (pos > 0 && value >= top->data[pos - 1].object.value) + pos--; + if (pos >= top->alloc) + return; + if (top->nr == top->alloc) + free(top->data[top->nr - 1].path); + else + top->nr++; + for (size_t i = top->nr - 1; i > pos; i--) + top->data[i] = top->data[i - 1]; + + oidcpy(&top->data[pos].object.oid, oid); + top->data[pos].object.value = value; + top->data[pos].path = xstrdup_or_null(path); + oidcpy(&top->data[pos].containing_commit_oid, + commit_oid ? commit_oid : null_oid(the_repository->hash_algo)); + /* Revision names are resolved only after ranking is complete. */ + top->data[pos].name_rev = NULL; +} + static size_t count_tree_entries(struct object *obj) { struct tree *t = object_as_type(obj, OBJ_TREE, 0); @@ -1065,6 +1446,19 @@ static size_t count_tree_entries(struct object *obj) return count; } +static void increment_histogram(struct object_histogram_bin *bins, + unsigned int shift, size_t value, + size_t inflated, off_t disk) +{ + size_t bin = 0; + + while (value >>= shift) + bin++; + bins[bin].count++; + bins[bin].inflated_size += inflated; + bins[bin].disk_size += disk; +} + static int count_objects(const char *path, struct oid_array *oids, enum object_type type, void *cb_data) { @@ -1075,7 +1469,7 @@ static int count_objects(const char *path, struct oid_array *oids, for (size_t i = 0; i < oids->nr; i++) { struct object_info oi = OBJECT_INFO_INIT; - size_t inflated; + size_t inflated, count; struct commit *commit; struct object *obj; void *content; @@ -1108,22 +1502,45 @@ static int count_objects(const char *path, struct oid_array *oids, break; case OBJ_COMMIT: commit = object_as_type(obj, OBJ_COMMIT, 0); + count = commit_list_count(commit->parents); stats->type_counts.commits++; stats->inflated_sizes.commits += inflated; stats->disk_sizes.commits += disk; check_largest(&stats->largest.commit_size, &oids->oid[i], inflated); check_largest(&stats->largest.parent_count, &oids->oid[i], - commit_list_count(commit->parents)); + count); + maybe_insert_top_object(&stats->top_commit_parents, + &oids->oid[i], count, NULL, + &oids->oid[i]); + maybe_insert_top_object(&stats->top_commit_sizes, + &oids->oid[i], inflated, NULL, + &oids->oid[i]); + if (count >= PBIN_VEC_LEN) + count = PBIN_VEC_LEN - 1; + stats->commit_parents[count]++; + increment_histogram(stats->commit_sizes, HBIN_SHIFT, + inflated, inflated, disk); break; case OBJ_TREE: + count = count_tree_entries(obj); stats->type_counts.trees++; stats->inflated_sizes.trees += inflated; stats->disk_sizes.trees += disk; check_largest(&stats->largest.tree_size, &oids->oid[i], inflated); check_largest(&stats->largest.tree_entries, &oids->oid[i], - count_tree_entries(obj)); + count); + maybe_insert_top_object(&stats->top_tree_entries, + &oids->oid[i], count, + path, NULL); + maybe_insert_top_object(&stats->top_tree_sizes, + &oids->oid[i], inflated, + path, NULL); + increment_histogram(stats->tree_entries, QBIN_SHIFT, + count, inflated, disk); + increment_histogram(stats->tree_sizes, HBIN_SHIFT, + inflated, inflated, disk); break; case OBJ_BLOB: stats->type_counts.blobs++; @@ -1131,6 +1548,11 @@ static int count_objects(const char *path, struct oid_array *oids, stats->disk_sizes.blobs += disk; check_largest(&stats->largest.blob_size, &oids->oid[i], inflated); + maybe_insert_top_object(&stats->top_blob_sizes, + &oids->oid[i], inflated, + path, NULL); + increment_histogram(stats->blob_sizes, HBIN_SHIFT, + inflated, inflated, disk); break; default: BUG("invalid object type"); @@ -1186,16 +1608,123 @@ static void structure_count_objects(struct object_stats *stats, stop_progress(&data.progress); } +static void structure_lookup_name_revs(struct object_stats *stats, + struct repository *repo, + int show_progress) +{ + struct top_objects *lists[] = { + &stats->top_commit_parents, + &stats->top_commit_sizes, + &stats->top_tree_entries, + &stats->top_tree_sizes, + &stats->top_blob_sizes, + }; + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf in = STRBUF_INIT, out = STRBUF_INIT; + struct string_list names = STRING_LIST_INIT_NODUP; + struct progress *progress = NULL; + size_t nr = 0, k = 0; + int failed = 1; + + for (size_t i = 0; i < ARRAY_SIZE(lists); i++) { + for (size_t j = 0; j < lists[i]->nr; j++) { + struct top_object *item = &lists[i]->data[j]; + + if (is_null_oid(&item->containing_commit_oid)) + continue; + strbuf_addf(&in, "%s\n", + oid_to_hex(&item->containing_commit_oid)); + nr++; + } + } + if (!nr) + return; + + trace2_region_enter("repo", "name-rev", repo); + if (show_progress) + progress = start_progress(repo, + _("Resolving revision names"), nr); + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "name-rev", "--name-only", + "--annotate-stdin", NULL); + if (pipe_command(&cp, in.buf, in.len, &out, 0, NULL, 0)) { + warning(_("could not resolve revision names")); + goto cleanup; + } + if (!out.len || out.buf[out.len - 1] != '\n' || + memchr(out.buf, '\0', out.len)) + goto invalid_output; + strbuf_trim_trailing_newline(&out); + string_list_split_in_place_f(&names, out.buf, "\n", -1, + STRING_LIST_SPLIT_TRIM); + if (names.nr != nr) + goto invalid_output; + for (size_t i = 0; i < names.nr; i++) + if (!*names.items[i].string) + goto invalid_output; + + for (size_t i = 0; i < ARRAY_SIZE(lists); i++) { + for (size_t j = 0; j < lists[i]->nr; j++) { + struct top_object *item = &lists[i]->data[j]; + + if (is_null_oid(&item->containing_commit_oid)) + continue; + item->name_rev = xstrdup(names.items[k++].string); + display_progress(progress, k); + } + } + failed = 0; + goto cleanup; + +invalid_output: + warning(_("unexpected output from 'git name-rev'")); +cleanup: + stop_progress_msg(&progress, failed ? _("failed") : _("done")); + trace2_region_leave("repo", "name-rev", repo); + string_list_clear(&names, 0); + strbuf_release(&in); + strbuf_release(&out); +} + +struct repo_structure_opts { + int name_rev; + int top_nr; + int commit_parents; + int commit_sizes; + int tree_entries; + int tree_sizes; + int blob_sizes; +}; + static int repo_structure_config_cb(const char *var, const char *value, const struct config_context *cctx, void *cb) { - int *top_nr = cb; + struct repo_structure_opts *opts = cb; + int *limit = NULL; + + if (!strcmp(var, "repo.structure.top")) + limit = &opts->top_nr; + else if (!strcmp(var, "repo.structure.showcommitparents")) + limit = &opts->commit_parents; + else if (!strcmp(var, "repo.structure.showcommitsizes")) + limit = &opts->commit_sizes; + else if (!strcmp(var, "repo.structure.showtreeentries")) + limit = &opts->tree_entries; + else if (!strcmp(var, "repo.structure.showtreesizes")) + limit = &opts->tree_sizes; + else if (!strcmp(var, "repo.structure.showblobsizes")) + limit = &opts->blob_sizes; + else if (!strcmp(var, "repo.structure.namerev")) { + opts->name_rev = git_config_bool(var, value); + return 0; + } - if (!strcmp(var, "repo.structure.top")) { - *top_nr = git_config_int(var, value, cctx->kvi); - if (*top_nr < 0) - die(_("repo.structure.top must be non-negative")); + if (limit) { + *limit = git_config_int(var, value, cctx->kvi); + if (*limit < 0) + die(_("%s must be non-negative"), var); return 0; } @@ -1211,9 +1740,9 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, }; enum output_format format = FORMAT_TABLE; struct repo_structure stats = { 0 }; + struct repo_structure_opts opts = { .name_rev = 1 }; struct rev_info revs; int show_progress = -1; - int top_nr = 0; struct string_list ref_filters = STRING_LIST_INIT_DUP; struct option options[] = { OPT_CALLBACK_F(0, "format", &format, N_("format"), @@ -1224,43 +1753,75 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, PARSE_OPT_NONEG | PARSE_OPT_NOARG, parse_format_cb), OPT_BOOL(0, "progress", &show_progress, N_("show progress")), + OPT_BOOL(0, "name-rev", &opts.name_rev, + N_("resolve revision names for reported commits")), OPT_STRING_LIST(0, "ref-filter", &ref_filters, N_("pattern"), N_("only count refs matching ; " "repeat to union multiple patterns")), - OPT_INTEGER(0, "top", &top_nr, + OPT_INTEGER(0, "top", &opts.top_nr, N_("report the top largest paths " "per category")), + OPT_INTEGER_F(0, "commit-parents", &opts.commit_parents, + N_("show commits with the most parents"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "commit-sizes", &opts.commit_sizes, + N_("show largest commits by size in bytes"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "tree-entries", &opts.tree_entries, + N_("show trees with the most entries"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "tree-sizes", &opts.tree_sizes, + N_("show largest trees by size in bytes"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "blob-sizes", &opts.blob_sizes, + N_("show largest blobs by size in bytes"), + PARSE_OPT_NONEG), OPT_END() }; - repo_config(repo, repo_structure_config_cb, &top_nr); + repo_config(repo, repo_structure_config_cb, &opts); argc = parse_options(argc, argv, prefix, options, repo_structure_usage, 0); if (argc) usage(_("too many arguments")); - if (top_nr < 0) + if (opts.top_nr < 0) die(_("--top= must be non-negative")); + init_top_objects(&stats.objects.top_commit_parents, + opts.commit_parents, "commit-parents"); + init_top_objects(&stats.objects.top_commit_sizes, + opts.commit_sizes, "commit-sizes"); + init_top_objects(&stats.objects.top_tree_entries, + opts.tree_entries, "tree-entries"); + init_top_objects(&stats.objects.top_tree_sizes, + opts.tree_sizes, "tree-sizes"); + init_top_objects(&stats.objects.top_blob_sizes, + opts.blob_sizes, "blob-sizes"); + repo_init_revisions(repo, &revs, prefix); if (show_progress < 0) show_progress = isatty(2); - if (top_nr) { - init_top_paths(&stats.objects.top_trees, top_nr); - init_top_paths(&stats.objects.top_blobs, top_nr); + if (opts.top_nr) { + init_top_paths(&stats.objects.top_trees, opts.top_nr); + init_top_paths(&stats.objects.top_blobs, opts.top_nr); } structure_count_references(&stats.refs, &revs, repo, &ref_filters, show_progress); - structure_count_objects(&stats.objects, &revs, repo, top_nr, + structure_count_objects(&stats.objects, &revs, repo, opts.top_nr, show_progress); + if (opts.name_rev) + structure_lookup_name_revs(&stats.objects, repo, show_progress); switch (format) { case FORMAT_TABLE: stats_table_setup_structure(&table, &stats); stats_table_setup_top_paths(&table, &stats.objects); - stats_table_print_structure(&table); + stats_table_print(&table, _("Repository structure")); + structure_histograms_table_print(&stats.objects); + structure_top_objects_table_print(&stats.objects); break; case FORMAT_NEWLINE_TERMINATED: structure_keyvalue_print(&stats, '=', '\n'); @@ -1274,10 +1835,15 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, stats_table_clear(&table); string_list_clear(&ref_filters, 0); - if (top_nr) { + if (opts.top_nr) { clear_top_paths(&stats.objects.top_trees); clear_top_paths(&stats.objects.top_blobs); } + clear_top_objects(&stats.objects.top_commit_parents); + clear_top_objects(&stats.objects.top_commit_sizes); + clear_top_objects(&stats.objects.top_tree_entries); + clear_top_objects(&stats.objects.top_tree_sizes); + clear_top_objects(&stats.objects.top_blob_sizes); release_revisions(&revs); return 0; diff --git a/builtin/reset.c b/builtin/reset.c index 66c26eab62fc12..cd107bc12fe088 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -40,6 +40,8 @@ #include "add-interactive.h" #include "strbuf.h" #include "quote.h" +#include "dir.h" +#include "entry.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) @@ -160,9 +162,54 @@ static void update_index_from_diff(struct diff_queue_struct *q, for (i = 0; i < q->nr; i++) { int pos; + int respect_skip_worktree = 1; struct diff_filespec *one = q->queue[i]->one; + struct diff_filespec *two = q->queue[i]->two; int is_in_reset_tree = one->mode && !is_null_oid(&one->oid); + int is_missing = !(one->mode && !is_null_oid(&one->oid)); + int was_missing = !two->mode && is_null_oid(&two->oid); struct cache_entry *ce; + struct cache_entry *ceBefore; + struct checkout state = CHECKOUT_INIT; + + /* + * When using the virtual filesystem feature, all entries + * being reset should have skip-worktree cleared so that + * refresh_index will compare them against the working tree + * and report them as modified. + * + * For files that don't exist on disk (virtual/placeholder), + * we also need to write the pre-reset content to disk so + * that they show as modified rather than deleted. + * + * For files that already exist on disk (hydrated), the + * on-disk content is the pre-reset version, so no write + * is needed — just clearing skip-worktree is sufficient. + */ + if (!core_virtualfilesystem) + ; /* not in virtual filesystem mode; nothing to special-case */ + else if (file_exists(two->path)) + respect_skip_worktree = 0; /* hydrated: on-disk content is already the pre-reset version */ + else { + respect_skip_worktree = 0; + pos = index_name_pos(the_repository->index, two->path, strlen(two->path)); + + if ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) && + (is_missing || !was_missing)) + { + state.force = 1; + state.refresh_cache = 1; + state.istate = the_repository->index; + ceBefore = make_cache_entry(the_repository->index, two->mode, + &two->oid, two->path, + 0, 0); + if (!ceBefore) + die(_("make_cache_entry failed for path '%s'"), + two->path); + + checkout_entry(ceBefore, &state, NULL, NULL); + } + } if (!is_in_reset_tree && !intent_to_add) { remove_file_from_index(the_repository->index, one->path); @@ -181,8 +228,14 @@ static void update_index_from_diff(struct diff_queue_struct *q, * to properly construct the reset sparse directory. */ pos = index_name_pos(the_repository->index, one->path, strlen(one->path)); - if ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) || - (pos < 0 && !path_in_sparse_checkout(one->path, the_repository->index))) + + /* + * Do not add the SKIP_WORKTREE bit back if we populated the + * file on purpose in a virtual filesystem scenario. + */ + if (respect_skip_worktree && + ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) || + (pos < 0 && !path_in_sparse_checkout(one->path, the_repository->index)))) ce->ce_flags |= CE_SKIP_WORKTREE; if (!ce) @@ -215,7 +268,8 @@ static int read_from_tree(const struct pathspec *pathspec, opt.add_remove = diff_addremove; if (pathspec->nr && pathspec_needs_expanded_index(the_repository->index, pathspec)) - ensure_full_index(the_repository->index); + ensure_full_index_with_reason(the_repository->index, + "reset pathspec"); if (do_diff_cache(tree_oid, &opt)) return 1; diff --git a/builtin/revert.c b/builtin/revert.c index bedc40f368eccc..bc50a8a51a5b5f 100644 --- a/builtin/revert.c +++ b/builtin/revert.c @@ -10,6 +10,7 @@ #include "rerere.h" #include "sequencer.h" #include "branch.h" +#include "config.h" /* * This implements the builtins revert and cherry-pick. @@ -288,6 +289,7 @@ int cmd_revert(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ opts.action = REPLAY_REVERT; sequencer_init_config(&opts); @@ -308,6 +310,7 @@ struct repository *repo UNUSED) #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ opts.action = REPLAY_PICK; sequencer_init_config(&opts); diff --git a/builtin/rm.c b/builtin/rm.c index 081d0bc3754c52..0d0e05e40dafd1 100644 --- a/builtin/rm.c +++ b/builtin/rm.c @@ -7,6 +7,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "environment.h" #include "advice.h" #include "config.h" #include "environment.h" @@ -310,12 +311,13 @@ int cmd_rm(int argc, seen = xcalloc(pathspec.nr, 1); if (pathspec_needs_expanded_index(the_repository->index, &pathspec)) - ensure_full_index(the_repository->index); + ensure_full_index_with_reason(the_repository->index, + "rm pathspec"); for (unsigned int i = 0; i < the_repository->index->cache_nr; i++) { const struct cache_entry *ce = the_repository->index->cache[i]; - if (!include_sparse && + if (!include_sparse && !core_virtualfilesystem && (ce_skip_worktree(ce) || !path_in_sparse_checkout(ce->name, the_repository->index))) continue; @@ -352,7 +354,11 @@ int cmd_rm(int argc, *original ? original : "."); } - if (only_match_skip_worktree.nr) { + /* + * When using a virtual filesystem, we might re-add a path + * that is currently virtual and we want that to succeed. + */ + if (!core_virtualfilesystem && only_match_skip_worktree.nr) { advise_on_updating_sparse_paths(&only_match_skip_worktree); ret = 1; } diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index cb4a037b770291..a2b0ef074a47f4 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -114,7 +114,7 @@ static int sparse_checkout_list(int argc, const char **argv, const char *prefix, static void clean_tracked_sparse_directories(struct repository *r) { - int i, was_full = 0; + int i, value, was_full = 0; struct strbuf path = STRBUF_INIT; size_t pathlen; struct string_list_item *item; @@ -130,6 +130,13 @@ static void clean_tracked_sparse_directories(struct repository *r) !r->index->sparse_checkout_patterns->use_cone_patterns) return; + /* + * Users can disable this behavior. + */ + if (!repo_config_get_bool(r, "index.deletesparsedirectories", &value) && + !value) + return; + /* * Use the sparse index as a data structure to assist finding * directories that are safe to delete. This conversion to a @@ -203,7 +210,8 @@ static void clean_tracked_sparse_directories(struct repository *r) strbuf_release(&path); if (was_full) - ensure_full_index(r->index); + ensure_full_index_with_reason(r->index, + "sparse-checkout:was full"); } static int update_working_directory(struct repository *r, @@ -438,7 +446,8 @@ static int update_modes(struct repository *repo, int *cone_mode, int *sparse_ind repo->index->updated_workdir = 1; if (!*sparse_index) - ensure_full_index(repo->index); + ensure_full_index_with_reason(repo->index, + "sparse-checkout:disabling sparse index"); } return 0; diff --git a/builtin/stash.c b/builtin/stash.c index 6e246a64bd323a..170024c4940b59 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -349,7 +349,7 @@ static int reset_tree(struct object_id *i_tree, int update, int reset) memset(&opts, 0, sizeof(opts)); tree = repo_parse_tree_indirect(the_repository, i_tree); - if (repo_parse_tree(the_repository, tree)) + if (!tree || repo_parse_tree(the_repository, tree)) return -1; init_tree_desc(t, &tree->object.oid, tree->buffer, tree->size); @@ -517,7 +517,7 @@ static int restore_untracked(struct object_id *u_tree) child_process_init(&cp); cp.git_cmd = 1; - strvec_pushl(&cp.args, "checkout-index", "--all", NULL); + strvec_pushl(&cp.args, "checkout-index", "--all", "-f", NULL); strvec_pushf(&cp.env, "GIT_INDEX_FILE=%s", stash_index_path.buf); @@ -1535,6 +1535,11 @@ static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_b goto done; } else { head_commit = lookup_commit(the_repository, &info->b_commit); + if (!head_commit) { + ret = error(_("could not look up commit '%s'"), + oid_to_hex (&info->b_commit)); + goto done; + } } if (!check_changes(ps, include_untracked, &untracked_files)) { diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index ebb9ff4477e154..2a29d649b8c486 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -3608,7 +3608,7 @@ static void die_on_index_match(const char *path, int force) char *ps_matched = xcalloc(ps.nr, 1); /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(the_repository->index); + ensure_full_index_unaudited(the_repository->index); /* * Since there is only one pathspec, we just need to diff --git a/builtin/survey.c b/builtin/survey.c index d3b36b891404cd..637dde74eb7c8b 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -111,3 +111,143 @@ int cmd_survey(int argc, const char **argv, const char *prefix, strvec_clear(&child_argv); return 1; } + +/* + * NEEDSWORK: So far, I only have iteration on the requested set of + * refs and treewalk/reachable objects on that set of refs. The + * following is a bit of a laundry list of things that I'd like to + * add. + * + * [] Dump stats on all of the packfiles. The number and size of each. + * Whether each is in the .git directory or in an alternate. The + * state of the IDX or MIDX files and etc. Delta chain stats. All + * of this data is relative to the "lived-in" state of the + * repository. Stuff that may change after a GC or repack. + * + * [] Clone and Index stats. partial, shallow, sparse-checkout, + * sparse-index, etc. Hydration stats. + * + * [] Dump stats on each remote. When we fetch from a remote the size + * of the response is related to the set of haves on the server. + * You can see this in `GIT_TRACE_CURL=1 git fetch`. We get a + * `ls-refs` payload that lists all of the branches and tags on the + * server, so at a minimum the RefName and SHA for each. But for + * annotated tags we also get the peeled SHA. The size of this + * overhead on every fetch is proporational to the size of the `git + * ls-remote` response (roughly, although the latter repeats the + * RefName of the peeled tag). If, for example, you have 500K refs + * on a remote, you're going to have a long "haves" message, so + * every fetch will be slow just because of that overhead (not + * counting new objects to be downloaded). + * + * Note that the local set of tags in "refs/tags/" is a union over + * all remotes. However, since most people only have one remote, + * we can probaly estimate the overhead value directly from the + * size of the set of "refs/tags/" that we visited while building + * the `ref_info` and `ref_array` and not need to ask the remote. + * + * [] Should the "string length of refnames / remote refs", for + * example, be sub-divided by remote so we can project the + * cost of the haves/wants overhead a fetch. + * + * [] Can we examine the merge commits and classify them as clean or + * dirty? (ie. ones with merge conflicts that needed to be + * addressed during the merge itself.) + * + * [] Do dirty merges affect performance of later operations? + * + * [] Dump info on the complexity of the DAG. Criss-cross merges. + * The number of edges that must be touched to compute merge bases. + * Edge length. The number of parallel lanes in the history that + * must be navigated to get to the merge base. What affects the + * cost of the Ahead/Behind computation? How often do + * criss-crosses occur and do they cause various operations to slow + * down? + * + * [] If there are primary branches (like "main" or "master") are they + * always on the left side of merges? Does the graph have a clean + * left edge? Or are there normal and "backwards" merges? Do + * these cause problems at scale? + * + * [] If we have a hierarchy of FI/RI branches like "L1", "L2, ..., + * can we learn anything about the shape of the repo around these + * FI and RI integrations? + * + * [] Do we need a no-PII flag to omit pathnames or branch/tag names + * in the various histograms? (This would turn off --name-rev + * too.) + * + * [] I have so far avoided adding opinions about individual fields + * (such as the way `git-sizer` prints a row of stars or bangs in + * the last column). + * + * I'm wondering if that is a job of this executable or if it + * should be done in a post-processing step using the JSON output. + * + * My problem with the `git-sizer` approach is that it doesn't give + * the (casual) user any information on why it has stars or bangs. + * And there isn't a good way to print detailed information in the + * ASCII-art tables that would be easy to understand. + * + * [] For example, a large number of refs does not define a cliff. + * Performance will drop off (linearly, quadratically, ... ??). + * The tool should refer them to article(s) talking about the + * different problems that it could cause. So should `git + * survey` just print the number and (implicitly) refer them to + * the man page (chapter/verse) or to a tool that will interpret + * the number and explain it? + * + * [] Alternatively, should `git survey` do that analysis too and + * just print footnotes for each large number? + * + * [] The computation of the raw survey JSON data can take HOURS on + * a very large repo (like Windows), so I'm wondering if we + * want to keep the opinion portion separate. + * + * [] In addition to opinions based on the static data, I would like + * to dump the JSON results (or the Trace2 telemetry) into a DB and + * aggregate it with other users. + * + * Granted, they should all see the same DAG and the same set of + * reachable objects, but we could average across all datasets + * generated on a particular date and detect outlier users. + * + * [] Maybe someone cloned from the `_full` endpoint rather than + * the limited refs endpoint. + * + * [] Maybe that user is having problems with repacking / GC / + * maintenance without knowing it. + * + * [] I'd also like to dump use the DB to compare survey datasets over + * a time. How fast is their repository growing and in what ways? + * + * [] I'd rather have the delta analysis NOT be inside `git + * survey`, so it makes sense to consider having all of it in a + * post-process step. + * + * [] Another reason to put the opinion analysis in a post-process + * is that it would be easier to generate plots on the data tables. + * Granted, we can get plots from telemetry, but a stand-alone user + * could run the JSON thru python or jq or something and generate + * something nicer than ASCII-art and it could handle cross-referencing + * and hyperlinking to helpful information on each issue. + * + * [] I think there are several classes of data that we can report on: + * + * [] The "inherit repo properties", such as the shape and size of + * the DAG -- these should be universal in each enlistment. + * + * [] The "ODB lived in properties", such as the efficiency + * of the repack and things like partial and shallow clone. + * These will vary, but indicate health of the ODB. + * + * [] The "index related properties", such as sparse-checkout, + * sparse-index, cache-tree, untracked-cache, fsmonitor, and + * etc. These will also vary, but are more like knobs for + * the user to adjust. + * + * [] I want to compare these with Matt's "dimensions of scale" + * notes and see if there are other pieces of data that we + * could compute/consider. + * + */ diff --git a/builtin/update-index.c b/builtin/update-index.c index b25d4ecb1091ba..9c89792780ba8a 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -8,6 +8,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "gvfs.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -706,7 +707,9 @@ static int do_reupdate(const char **paths, * to process each path individually */ if (S_ISSPARSEDIR(ce->ce_mode)) { - ensure_full_index(the_repository->index); + const char *fmt = "update-index:modified sparse dir '%s'"; + ensure_full_index_with_reason(the_repository->index, + fmt, ce->name); goto redo; } @@ -915,7 +918,7 @@ static enum parse_opt_result reupdate_callback( int cmd_update_index(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int newfd, entries, has_errors = 0, nul_term_line = 0; enum uc_mode untracked_cache = UC_UNSPECIFIED; @@ -1181,7 +1184,13 @@ int cmd_update_index(int argc, argc = parse_options_end(&ctx); getline_fn = nul_term_line ? strbuf_getline_nul : strbuf_getline_lf; + if (mark_skip_worktree_only && gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die(_("modifying the skip worktree bit is not supported on a GVFS repo")); + if (preferred_index_format) { + if (preferred_index_format != 4 && gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die(_("changing the index version is not supported on a GVFS repo")); + if (preferred_index_format < 0) { printf(_("%d\n"), the_repository->index->version); } else if (preferred_index_format < INDEX_FORMAT_LB || @@ -1227,6 +1236,9 @@ int cmd_update_index(int argc, odb_transaction_commit_and_finalize_or_die(transaction); if (split_index > 0) { + if (gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die(_("split index is not supported on a GVFS repo")); + if (repo_config_get_split_index(the_repository) == 0) warning(_("core.splitIndex is set to false; " "remove or change it, if you really want to " diff --git a/builtin/update-microsoft-git.c b/builtin/update-microsoft-git.c new file mode 100644 index 00000000000000..357cd598962046 --- /dev/null +++ b/builtin/update-microsoft-git.c @@ -0,0 +1,76 @@ +#include "builtin.h" +#include "repository.h" +#include "parse-options.h" +#include "run-command.h" +#include "strvec.h" + +#if defined(GIT_WINDOWS_NATIVE) +/* + * On Windows, run 'git update-git-for-windows' which + * is installed by the installer, based on the script + * in git-for-windows/build-extra. + */ +static int platform_specific_upgrade(void) +{ + struct child_process cp = CHILD_PROCESS_INIT; + + strvec_push(&cp.args, "git-update-git-for-windows"); + return run_command(&cp); +} +#elif defined(__APPLE__) +/* + * On macOS, we expect the user to have the microsoft-git + * cask installed via Homebrew. We check using these + * commands: + * + * 1. 'brew update' to get latest versions. + * 2. 'brew upgrade --cask microsoft-git' to get the + * latest version. + */ +static int platform_specific_upgrade(void) +{ + int res; + struct child_process update = CHILD_PROCESS_INIT; + struct child_process upgrade = CHILD_PROCESS_INIT; + + printf("Updating Homebrew with 'brew update'\n"); + + strvec_pushl(&update.args, "brew", "update", NULL); + res = run_command(&update); + + if (res) { + error(_("'brew update' failed; is brew installed?")); + return 1; + } + + printf("Upgrading microsoft-git with 'brew upgrade --cask microsoft-git'\n"); + strvec_pushl(&upgrade.args, "brew", "upgrade", "--cask", "microsoft-git", NULL); + res = run_command(&upgrade); + + return res; +} +#else +static int platform_specific_upgrade(void) +{ + error(_("update-microsoft-git is not supported on this platform")); + return 1; +} +#endif + +static const char * const update_microsoft_git_usage[] = { + N_("git update-microsoft-git"), + NULL, +}; + + +int cmd_update_microsoft_git(int argc, const char **argv, const char *prefix UNUSED, struct repository *repo UNUSED) +{ + static struct option microsoft_git_options[] = { + OPT_END(), + }; + show_usage_with_options_if_asked(argc, argv, + update_microsoft_git_usage, + microsoft_git_options); + + return platform_specific_upgrade(); +} diff --git a/builtin/worktree.c b/builtin/worktree.c index 77ecd0f71f4247..eda5dc239a2057 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -4,6 +4,7 @@ #include "builtin.h" #include "abspath.h" #include "advice.h" +#include "gvfs.h" #include "checkout.h" #include "config.h" #include "copy.h" @@ -886,6 +887,14 @@ static int add(int ac, const char **av, const char *prefix, if (ac < 1 || ac > 2) usage_with_options(git_worktree_add_usage, options); + /* + * When the virtual file system is active, skip checkout during + * worktree creation. The VFS layer will handle the checkout + * after the worktree structure is set up. + */ + if (gvfs_config_is_set(the_repository, GVFS_USE_VIRTUAL_FILESYSTEM)) + opts.checkout = 0; + path = prefix_filename(prefix, av[0]); branch = ac < 2 ? "HEAD" : av[1]; used_new_branch_options = new_branch || new_branch_force; @@ -1422,6 +1431,21 @@ static int delete_git_work_tree(struct worktree *wt) return ret; } +/* + * Check if a pre-command hook has already verified worktree cleanliness + * and written a marker file to skip git's own check. VFSForGit uses this + * to unmount ProjFS after its own status check; without it, git's status + * call would fail because the virtual filesystem is no longer available. + */ +static int should_skip_clean_check(struct worktree *wt) +{ + char *path = repo_common_path(the_repository, + "worktrees/%s/skip-clean-check", wt->id); + int skip = file_exists(path); + free(path); + return skip; +} + static int remove_worktree(int ac, const char **av, const char *prefix, struct repository *repo UNUSED) { @@ -1461,7 +1485,9 @@ static int remove_worktree(int ac, const char **av, const char *prefix, strbuf_release(&errmsg); if (file_exists(wt->path)) { - if (!force) + if (!force && + !(gvfs_config_is_set(the_repository, GVFS_SUPPORTS_WORKTREES) && + should_skip_clean_check(wt))) check_clean_worktree(wt, av[0]); ret |= delete_git_work_tree(wt); @@ -1534,6 +1560,14 @@ int cmd_worktree(int ac, ac = parse_options(ac, av, prefix, options, git_worktree_usage, 0); + /* + * Block worktree commands when VFS is active unless the VFS layer + * has signaled worktree support via GVFS_SUPPORTS_WORKTREES. + */ + if (gvfs_config_is_set(the_repository, GVFS_USE_VIRTUAL_FILESYSTEM) && + !gvfs_config_is_set(the_repository, GVFS_SUPPORTS_WORKTREES)) + die("'git worktree' is not supported when using the virtual file system"); + prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/bundle.c b/bundle.c index f55a521b2a1f12..d371fd7a43101d 100644 --- a/bundle.c +++ b/bundle.c @@ -72,7 +72,7 @@ static int parse_bundle_signature(struct bundle_header *header, const char *line int i; for (i = 0; i < ARRAY_SIZE(bundle_sigs); i++) { - if (!strcmp(line, bundle_sigs[i].signature)) { + if (!strcmp(line, bundle_sigs[i].signature)) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand header->version = bundle_sigs[i].version; return 0; } @@ -88,7 +88,7 @@ int read_bundle_header_fd(int fd, struct bundle_header *header, /* The bundle header begins with the signature */ if (strbuf_getwholeline_fd(&buf, fd, '\n') || - parse_bundle_signature(header, buf.buf)) { + parse_bundle_signature(header, buf.buf)) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand if (report_path) error(_("'%s' does not look like a v2 or v3 bundle file"), report_path); diff --git a/cache-tree.c b/cache-tree.c index b8cbb5da221020..defdda314b0a0c 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" +#include "gvfs.h" #include "lockfile.h" #include "tree.h" #include "tree-walk.h" @@ -274,8 +275,8 @@ static void discard_unused_subtrees(struct cache_tree *it) } } -static int cache_tree_fully_valid_recursive(struct object_database *odb, - struct cache_tree *it) +static int cache_tree_fully_valid_recursive_1(struct object_database *odb, + struct cache_tree *it) { int i; if (!it) @@ -285,12 +286,24 @@ static int cache_tree_fully_valid_recursive(struct object_database *odb, ODB_HAS_OBJECT_RECHECK_PACKED | ODB_HAS_OBJECT_FETCH_PROMISOR)) return 0; for (i = 0; i < it->subtree_nr; i++) { - if (!cache_tree_fully_valid_recursive(odb, it->down[i]->cache_tree)) + if (!cache_tree_fully_valid_recursive_1(odb, it->down[i]->cache_tree)) return 0; } return 1; } +static int cache_tree_fully_valid_recursive(struct object_database *odb, + struct cache_tree *it) +{ + int result; + + trace2_region_enter("cache_tree", "fully_valid", NULL); + result = cache_tree_fully_valid_recursive_1(odb, it); + trace2_region_leave("cache_tree", "fully_valid", NULL); + + return result; +} + int cache_tree_fully_valid(struct index_state *istate) { return cache_tree_fully_valid_recursive(istate->repo->objects, @@ -313,7 +326,8 @@ static int update_one(struct repository *repo, int flags) { struct strbuf buffer; - int missing_ok = flags & WRITE_TREE_MISSING_OK; + int missing_ok = gvfs_config_is_set(repo, GVFS_MISSING_OK) ? + WRITE_TREE_MISSING_OK : (flags & WRITE_TREE_MISSING_OK); int dryrun = flags & WRITE_TREE_DRY_RUN; int repair = flags & WRITE_TREE_REPAIR; int to_invalidate = 0; @@ -487,7 +501,29 @@ static int update_one(struct repository *repo, continue; strbuf_grow(&buffer, entlen + 100); - strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); + + switch (mode) { + case 0100644: + strbuf_add(&buffer, "100644 ", 7); + break; + case 0100664: + strbuf_add(&buffer, "100664 ", 7); + break; + case 0100755: + strbuf_add(&buffer, "100755 ", 7); + break; + case 0120000: + strbuf_add(&buffer, "120000 ", 7); + break; + case 0160000: + strbuf_add(&buffer, "160000 ", 7); + break; + default: + strbuf_addf(&buffer, "%o ", mode); + break; + } + strbuf_add(&buffer, path + baselen, entlen); + strbuf_addch(&buffer, '\0'); strbuf_add(&buffer, oid->hash, repo->hash_algo->rawsz); #if DEBUG_CACHE_TREE diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 8606e30779b7fe..24f91fca4b9096 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -145,7 +145,7 @@ case "$jobname" in ClangFormat) sudo apt-get -q -y install clang-format ;; -StaticAnalysis) +StaticAnalysis|codeql) sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext make ;; diff --git a/commit-graph.c b/commit-graph.c index 983c11ce853459..25b325a1ee4ac9 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -2607,6 +2607,7 @@ int write_commit_graph(struct odb_source *source, g = prepare_commit_graph(ctx.r); for (struct commit_graph *chain = g; chain; chain = chain->base_graph) + /* Intentional: codeql[cpp/stack-address-escape] */ chain->topo_levels = &topo_levels; if (flags & COMMIT_GRAPH_WRITE_BLOOM_FILTERS) @@ -2830,6 +2831,11 @@ static int verify_one_commit_graph(struct commit_graph *g, g->hash_algo); graph_commit = lookup_commit(r, &cur_oid); + if (!graph_commit) { + graph_report(_("failed to look up commit %s for commit-graph"), + oid_to_hex(&cur_oid)); + continue; + } odb_commit = (struct commit *)create_object(r, &cur_oid, alloc_commit_node(r)); if (repo_parse_commit_internal(r, odb_commit, 0, 0)) { graph_report(_("failed to parse commit %s from object database for commit-graph"), diff --git a/commit.c b/commit.c index 9ba5af99eae693..79179180553d9d 100644 --- a/commit.c +++ b/commit.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "gvfs.h" #include "tag.h" #include "commit.h" #include "commit-graph.h" @@ -210,7 +211,7 @@ void unparse_commit(struct repository *r, const struct object_id *oid) { struct commit *c = lookup_commit(r, oid); - if (!c->object.parsed) + if (!c || !c->object.parsed) return; commit_list_free(c->parents); c->parents = NULL; @@ -613,13 +614,22 @@ int repo_parse_commit_internal(struct repository *r, .sizep = &size, .contentp = &buffer, }; + int ret; /* * Git does not support partial clones that exclude commits, so set * OBJECT_INFO_SKIP_FETCH_OBJECT to fail fast when an object is missing. */ int flags = OBJECT_INFO_LOOKUP_REPLACE | OBJECT_INFO_SKIP_FETCH_OBJECT | - OBJECT_INFO_DIE_IF_CORRUPT; - int ret; + OBJECT_INFO_DIE_IF_CORRUPT; + + /* + * But the GVFS Protocol _does_ support missing commits! + * And the idea with VFS for Git is to re-download corrupted objects, + * not to fail! + */ + if (gvfs_config_is_set(r, GVFS_MISSING_OK)) + flags &= ~(OBJECT_INFO_SKIP_FETCH_OBJECT | + OBJECT_INFO_DIE_IF_CORRUPT); if (!item) return -1; diff --git a/compat/mingw.c b/compat/mingw.c index fd014eef498e55..fbd118b5937a35 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -4422,6 +4422,8 @@ int wmain(int argc, const wchar_t **wargv) SetConsoleCtrlHandler(handle_ctrl_c, TRUE); + trace2_initialize_clock(); + maybe_redirect_std_handles(); adjust_symlink_flags(); fsync_object_files = 1; diff --git a/config.c b/config.c index d9019e7e6c34b0..f65b867c92d0b5 100644 --- a/config.c +++ b/config.c @@ -10,6 +10,7 @@ #include "abspath.h" #include "advice.h" #include "date.h" +#include "gvfs.h" #include "branch.h" #include "config.h" #include "dir.h" @@ -36,6 +37,7 @@ #include "trace2.h" #include "wildmatch.h" #include "write-or-die.h" +#include "transport.h" struct config_source { struct config_source *prev; @@ -2539,6 +2541,46 @@ int repo_config_get_max_percent_split_change(struct repository *r) return -1; /* default value */ } +int repo_config_get_virtualfilesystem(struct repository *r) +{ + /* Run only once. */ + static int virtual_filesystem_result = -1; + struct repo_config_values *cfg = repo_config_values(r); + extern char *core_virtualfilesystem; + if (virtual_filesystem_result >= 0) + return virtual_filesystem_result; + + if (repo_config_get_pathname(r, "core.virtualfilesystem", &core_virtualfilesystem)) + core_virtualfilesystem = xstrdup_or_null(getenv("GIT_VIRTUALFILESYSTEM_TEST")); + + if (core_virtualfilesystem && !*core_virtualfilesystem) + FREE_AND_NULL(core_virtualfilesystem); + + if (core_virtualfilesystem) { + /* + * Some git commands spawn helpers and redirect the index to a different + * location. These include "difftool -d" and the sequencer + * (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others. + * In those instances we don't want to update their temporary index with + * our virtualization data. + */ + char *default_index_file = xstrfmt("%s/%s", r->gitdir, "index"); + int should_run_hook = !strcmp(default_index_file, r->index_file); + + free(default_index_file); + if (should_run_hook) { + /* virtual file system relies on the sparse checkout logic so force it on */ + cfg->apply_sparse_checkout = 1; + virtual_filesystem_result = 1; + return 1; + } + FREE_AND_NULL(core_virtualfilesystem); + } + + virtual_filesystem_result = 0; + return 0; +} + int repo_config_get_index_threads(struct repository *r, int *dest) { int is_bool, val; @@ -2974,7 +3016,20 @@ static long config_lock_timeout_ms(struct repository *r) static int timeout_ms = 1000; if (!configured) { - repo_config_get_int(r, "core.configlocktimeout", &timeout_ms); + if (repo_config_get_int(r, "core.configlocktimeout", &timeout_ms) && + /* + * If 'core.configWriteLockTimeoutMS' is set, print a + * deprecation warning suggesting the use of + * 'core.configLockTimeout' instead. + */ + !repo_config_get_int(r, "core.configWriteLockTimeoutMS", + &timeout_ms) && + !git_env_bool("GIT_SUPPRESS_CONFIG_WRITE_LOCK_TIMEOUT_MS_ADVICE", 0)) { + advise_if_enabled(ADVICE_USE_CORE_CONFIG_WRITE_LOCK_TIMEOUT_MS_CONFIG, + _("core.configWriteLockTimeoutMS is deprecated;" + "please set core.configLockTimeout instead")); + setenv("GIT_SUPPRESS_CONFIG_WRITE_LOCK_TIMEOUT_MS_ADVICE", "1", 1); + } configured = 1; } diff --git a/config.h b/config.h index b66dd08007c97a..c6075701e728be 100644 --- a/config.h +++ b/config.h @@ -703,6 +703,8 @@ int repo_config_get_index_threads(struct repository *r, int *dest); int repo_config_get_split_index(struct repository *r); int repo_config_get_max_percent_split_change(struct repository *r); +int repo_config_get_virtualfilesystem(struct repository *r); + /* This dies if the configured or default date is in the future */ int repo_config_get_expiry(struct repository *r, const char *key, char **output); diff --git a/connected.c b/connected.c index 929b9bd28d6fab..023ae258bbfe7b 100644 --- a/connected.c +++ b/connected.c @@ -1,7 +1,9 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "environment.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "odb.h" #include "run-command.h" @@ -90,6 +92,26 @@ int check_connected(oid_iterate_fn fn, void *cb_data, struct transport *transport; size_t base_len; + /* + * Running a virtual file system there will be objects that are + * missing locally and we don't want to download a bunch of + * commits, trees, and blobs just to make sure everything is + * reachable locally so this option will skip reachablility + * checks below that use rev-list. This will stop the check + * before uploadpack runs to determine if there is anything to + * fetch. Returning zero for the first check will also prevent the + * uploadpack from happening. It will also skip the check after + * the fetch is finished to make sure all the objects where + * downloaded in the pack file. This will allow the fetch to + * run and get all the latest tip commit ids for all the branches + * in the fetch but not pull down commits, trees, or blobs via + * upload pack. + */ + if (gvfs_config_is_set(the_repository, GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) + return 0; + if (gvfs_virtualize_objects(the_repository)) + return 0; + if (!opt) opt = &defaults; transport = opt->transport; diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 524b3aad9247d3..e39a85bd97dc0a 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -258,7 +258,7 @@ add_compile_definitions(PAGER_ENV="LESS=FRX LV=-c" BINDIR="bin") if(WIN32) - set(FALLBACK_RUNTIME_PREFIX /mingw64) + set(FALLBACK_RUNTIME_PREFIX /ucrt64) # Move system config into top-level /etc/ add_compile_definitions(FALLBACK_RUNTIME_PREFIX="${FALLBACK_RUNTIME_PREFIX}" ETC_GITATTRIBUTES="../etc/gitattributes" @@ -646,7 +646,7 @@ if(NOT CURL_FOUND) add_compile_definitions(NO_CURL) message(WARNING "git-http-push and git-http-fetch will not be built") else() - list(APPEND PROGRAMS_BUILT git-http-fetch git-http-push git-imap-send git-remote-http) + list(APPEND PROGRAMS_BUILT git-http-fetch git-http-push git-imap-send git-remote-http git-gvfs-helper) if(CURL_VERSION_STRING VERSION_GREATER_EQUAL 7.34.0) add_compile_definitions(USE_CURL_FOR_IMAP_SEND) endif() @@ -803,7 +803,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main) add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c) target_link_libraries(git-shell common-main) -add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c) +add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c) target_link_libraries(scalar common-main) if(CURL_FOUND) @@ -822,6 +822,9 @@ if(CURL_FOUND) add_executable(git-http-push ${CMAKE_SOURCE_DIR}/http-push.c) target_link_libraries(git-http-push http_obj common-main ${CURL_LIBRARIES} ${EXPAT_LIBRARIES}) endif() + + add_executable(git-gvfs-helper ${CMAKE_SOURCE_DIR}/gvfs-helper.c) + target_link_libraries(git-gvfs-helper http_obj common-main ${CURL_LIBRARIES} ) endif() parse_makefile_for_executables(git_builtin_extra "BUILT_INS") @@ -1121,6 +1124,20 @@ set(wrapper_scripts set(wrapper_test_scripts test-fake-ssh test-tool) +if(CURL_FOUND) + list(APPEND wrapper_test_scripts test-gvfs-protocol) + + add_executable(test-gvfs-protocol ${CMAKE_SOURCE_DIR}/t/helper/test-gvfs-protocol.c) + target_link_libraries(test-gvfs-protocol common-main) + + if(MSVC) + set_target_properties(test-gvfs-protocol + PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}/t/helper) + set_target_properties(test-gvfs-protocol + PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/t/helper) + endif() +endif() + foreach(script ${wrapper_scripts}) file(STRINGS ${CMAKE_SOURCE_DIR}/bin-wrappers/wrap-for-bin.sh content NEWLINE_CONSUME) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 9f8b9b50ff834c..ee64f26e5f1ee5 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1848,7 +1848,7 @@ _git_clone () esac } -__git_untracked_file_modes="all no normal" +__git_untracked_file_modes="all no normal complete" __git_trailer_tokens () { diff --git a/contrib/long-running-read-object/example.pl b/contrib/long-running-read-object/example.pl new file mode 100644 index 00000000000000..b8f37f836a813c --- /dev/null +++ b/contrib/long-running-read-object/example.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "/host_repo/.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/contrib/scalar/docs/faq.md b/contrib/scalar/docs/faq.md new file mode 100644 index 00000000000000..a14f78a996d5d5 --- /dev/null +++ b/contrib/scalar/docs/faq.md @@ -0,0 +1,51 @@ +Frequently Asked Questions +========================== + +Using Scalar +------------ + +### I don't want a sparse clone, I want every file after I clone! + +Run `scalar clone --full-clone ` to initialize your repo to include +every file. You can switch to a sparse-checkout later by running +`git sparse-checkout init --cone`. + +### I already cloned without `--full-clone`. How do I get everything? + +Run `git sparse-checkout disable`. + +Scalar Design Decisions +----------------------- + +There may be many design decisions within Scalar that are confusing at first +glance. Some of them may cause friction when you use Scalar with your existing +repos and existing habits. + +> Scalar has the most benefit when users design repositories +> with efficient patterns. + +For example: Scalar uses the sparse-checkout feature to limit the size of the +working directory within a large monorepo. It is designed to work efficiently +with monorepos that are highly componentized, allowing most developers to +need many fewer files in their daily work. + +### Why does `scalar clone` create a `/src` folder? + +Scalar uses a file system watcher to keep track of changes under this `src` folder. +Any activity in this folder is assumed to be important to Git operations. By +creating the `src` folder, we are making it easy for your build system to +create output folders outside the `src` directory. We commonly see systems +create folders for build outputs and package downloads. Scalar itself creates +these folders during its builds. + +Your build system may create build artifacts such as `.obj` or `.lib` files +next to your source code. These are commonly "hidden" from Git using +`.gitignore` files. Having such artifacts in your source tree creates +additional work for Git because it needs to look at these files and match them +against the `.gitignore` patterns. + +By following the `src` pattern Scalar tries to establish and placing your build +intermediates and outputs parallel with the `src` folder and not inside it, +you can help optimize Git command performance for developers in the repository +by limiting the number of files Git needs to consider for many common +operations. diff --git a/contrib/scalar/docs/getting-started.md b/contrib/scalar/docs/getting-started.md new file mode 100644 index 00000000000000..d5125330320d2c --- /dev/null +++ b/contrib/scalar/docs/getting-started.md @@ -0,0 +1,109 @@ +Getting Started +=============== + +Registering existing Git repos +------------------------------ + +To add a repository to the list of registered repos, run `scalar register []`. +If `` is not provided, then the "current repository" is discovered from +the working directory by scanning the parent paths for a path containing a `.git` +folder, possibly inside a `src` folder. + +To see which repositories are currently tracked by the service, run +`scalar list`. + +Run `scalar unregister []` to remove the repo from this list. + +Creating a new Scalar clone +--------------------------------------------------- + +The `clone` verb creates a local enlistment of a remote repository using the +partial clone feature available e.g. on GitHub, or using the +[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md), +such as Azure Repos. + +``` +scalar clone [options] [] +``` + +Create a local copy of the repository at ``. If specified, create the `` +directory and place the repository there. Otherwise, the last section of the `` +will be used for ``. + +At the end, the repo is located at `/src`. By default, the sparse-checkout +feature is enabled and the only files present are those in the root of your +Git repository. Use `git sparse-checkout set` to expand the set of directories +you want to see, or `git sparse-checkout disable` to expand to all files. You +can explore the subdirectories outside your sparse-checkout specification using +`git ls-tree HEAD`. + +### Sparse Repo Mode + +By default, Scalar reduces your working directory to only the files at the +root of the repository. You need to add the folders you care about to build up +to your working set. + +* `scalar clone ` + * Please choose the **Clone with HTTPS** option in the `Clone Repository` dialog in Azure Repos, not **Clone with SSH**. +* `cd \src` +* At this point, your `src` directory only contains files that appear in your root + tree. No folders are populated. +* Set the directory list for your sparse-checkout using: + 1. `git sparse-checkout set ...` + 2. `git sparse-checkout set --stdin < dir-list.txt` +* Run git commands as you normally would. +* To fully populate your working directory, run `git sparse-checkout disable`. + +If instead you want to start with all files on-disk, you can clone with the +`--full-clone` option. To enable sparse-checkout after the fact, run +`git sparse-checkout init --cone`. This will initialize your sparse-checkout +patterns to only match the files at root. + +If you are unfamiliar with what directories are available in the repository, +then you can run `git ls-tree -d --name-only HEAD` to discover the directories +at root, or `git ls-tree -d --name-only HEAD ` to discover the directories +in ``. + +### Options + +These options allow a user to customize their initial enlistment. + +* `--full-clone`: If specified, do not initialize the sparse-checkout feature. + All files will be present in your `src` directory. This behaves very similar + to a Git partial clone in that blobs are downloaded on demand. However, it + will use the GVFS protocol to download all Git objects. + +* `--cache-server-url=`: If specified, set the intended cache server to + the specified ``. All object queries will use the GVFS protocol to this + `` instead of the origin remote. If the remote supplies a list of + cache servers via the `/gvfs/config` endpoint, then the `clone` command + will select a nearby cache server from that list. + +* `--branch=`: Specify the branch to checkout after clone. + +* `--local-cache-path=`: Use this option to override the path for the + local Scalar cache. If not specified, then Scalar will select a default + path to share objects with your other enlistments. On Windows, this path + is a subdirectory of `:\.scalarCache\`. On Mac, this path is a + subdirectory of `~/.scalarCache/`. The default cache path is recommended so + multiple enlistments of the same remote repository share objects on the + same device. + +### Advanced Options + +The options below are not intended for use by a typical user. These are +usually used by build machines to create a temporary enlistment that +operates on a single commit. + +* `--single-branch`: Use this option to only download metadata for the branch + that will be checked out. This is helpful for build machines that target + a remote with many branches. Any `git fetch` commands after the clone will + still ask for all branches. + +Removing a Scalar Clone +----------------------- + +Since the `scalar clone` command sets up a file-system watcher (when available), +that watcher could prevent deleting the enlistment. Run `scalar delete ` +from outside of your enlistment to unregister the enlistment from the filesystem +watcher and delete the enlistment at ``. diff --git a/contrib/scalar/docs/index.md b/contrib/scalar/docs/index.md new file mode 100644 index 00000000000000..4f56e2b0ebbac6 --- /dev/null +++ b/contrib/scalar/docs/index.md @@ -0,0 +1,54 @@ +Scalar: Enabling Git at Scale +============================= + +Scalar is a tool that helps Git scale to some of the largest Git repositories. +It achieves this by enabling some advanced Git features, such as: + +* *Partial clone:* reduces time to get a working repository by not + downloading all Git objects right away. + +* *Background prefetch:* downloads Git object data from all remotes every + hour, reducing the amount of time for foreground `git fetch` calls. + +* *Sparse-checkout:* limits the size of your working directory. + +* *File system monitor:* tracks the recently modified files and eliminates + the need for Git to scan the entire worktree. + +* *Commit-graph:* accelerates commit walks and reachability calculations, + speeding up commands like `git log`. + +* *Multi-pack-index:* enables fast object lookups across many pack-files. + +* *Incremental repack:* Repacks the packed Git data into fewer pack-file + without disrupting concurrent commands by using the multi-pack-index. + +By running `scalar register` in any Git repo, Scalar will automatically enable +these features for that repo (except partial clone) and start running suggested +maintenance in the background using +[the `git maintenance` feature](https://git-scm.com/docs/git-maintenance). + +Repos cloned with the `scalar clone` command use partial clone or the +[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md) +to significantly reduce the amount of data required to get started +using a repository. By delaying all blob downloads until they are required, +Scalar allows you to work with very large repositories quickly. The GVFS +protocol allows a network of _cache servers_ to serve objects with lower +latency and higher throughput. The cache servers also reduce load on the +central server. + +Documentation +------------- + +* [Getting Started](getting-started.md): Get started with Scalar. + Includes `scalar register`, `scalar unregister`, `scalar clone`, and + `scalar delete`. + +* [Troubleshooting](troubleshooting.md): + Collect diagnostic information or update custom settings. Includes + `scalar diagnose` and `scalar cache-server`. + +* [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way + it does, and how do we make decisions about its future? + +* [Frequently Asked Questions](faq.md) diff --git a/contrib/scalar/docs/philosophy.md b/contrib/scalar/docs/philosophy.md new file mode 100644 index 00000000000000..e3dfa025a2504c --- /dev/null +++ b/contrib/scalar/docs/philosophy.md @@ -0,0 +1,71 @@ +The Philosophy of Scalar +======================== + +The team building Scalar has **opinions** about Git performance. Scalar +takes out the guesswork by automatically configuring your Git repositories +to take advantage of the latest and greatest features. It is difficult to +say that these are the absolute best settings for every repository, but +these settings do work for some of the largest repositories in the world. + +Scalar intends to do very little more than the standard Git client. We +actively implement new features into Git instead of Scalar, then update +Scalar only to configure those new settings. In particular, we ported +features like background maintenance to Git to make Scalar simpler and +make Git more powerful. + +Scalar ships inside [a custom version of Git][microsoft-git], but we are +working to make it available in other forks of Git. The only feature +that is not intended to ever reach the standard Git client is Scalar's use +of [the GVFS Protocol][gvfs-protocol], which is essentially an older +version of [Git's partial clone feature](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/) +that was available first in Azure Repos. Services such as GitHub support +only partial clone instead of the GVFS protocol because that is the +standard adopted by the Git project. If your hosting service supports +partial clone, then we absolutely recommend it as a way to greatly speed +up your clone and fetch times and to reduce how much disk space your Git +repository requires. Scalar will help with this! + +If you don't use the GVFS Protocol, then most of the value of Scalar can +be found in the core Git client. However, most of the advanced features +that really optimize Git's performance are off by default for compatibility +reasons. To really take advantage of Git's latest and greatest features, +you either need to study the [`git config` documentation](https://git-scm.com/docs/git-config) +and regularly read [the Git release notes](https://github.com/git/git/tree/master/Documentation/RelNotes). +Even if you do all that work and customize your Git settings on your machines, +you likely will want to share those settings with other team members. +Or, you can just use Scalar! + +Using `scalar register` on an existing Git repository will give you these +benefits: + +* Additional compression of your `.git/index` file. +* Hourly background `git fetch` operations, keeping you in-sync with your + remotes. +* Advanced data structures, such as the `commit-graph` and `multi-pack-index` + are updated automatically in the background. +* If using macOS or Windows, then Scalar configures Git's builtin File System + Monitor, providing faster commands such as `git status` or `git add`. + +Additionally, if you use `scalar clone` to create a new repository, then +you will automatically get these benefits: + +* Use Git's partial clone feature to only download the files you need for + your current checkout. +* Use Git's [sparse-checkout feature][sparse-checkout] to minimize the + number of files required in your working directory. + [Read more about sparse-checkout here.][sparse-checkout-blog] +* Create the Git repository inside `/src` to make it easy to + place build artifacts outside of the Git repository, such as in + `/bin` or `/packages`. + +We also admit that these **opinions** can always be improved! If you have +an idea of how to improve our setup, consider +[creating an issue](https://github.com/microsoft/scalar/issues/new) or +contributing a pull request! Some [existing](https://github.com/microsoft/scalar/issues/382) +[issues](https://github.com/microsoft/scalar/issues/388) have already +improved our configuration settings and roadmap! + +[gvfs-protocol]: https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md +[microsoft-git]: https://github.com/microsoft/git +[sparse-checkout]: https://git-scm.com/docs/git-sparse-checkout +[sparse-checkout-blog]: https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/ diff --git a/contrib/scalar/docs/troubleshooting.md b/contrib/scalar/docs/troubleshooting.md new file mode 100644 index 00000000000000..c54d2438f22523 --- /dev/null +++ b/contrib/scalar/docs/troubleshooting.md @@ -0,0 +1,40 @@ +Troubleshooting +=============== + +Diagnosing Issues +----------------- + +The `scalar diagnose` command collects logs and config details for the current +repository. The resulting zip file helps root-cause issues. + +When run inside your repository, creates a zip file containing several important +files for that repository. This includes: + +* Configuration files from your `.git` folder, such as the `config` file, + `index`, `hooks`, and `refs`. + +* A summary of your Git object database, including the number of loose objects + and the names and sizes of pack-files. + +As the `diagnose` command completes, it provides the path of the resulting +zip file. This zip can be attached to bug reports to make the analysis easier. + +Modifying Configuration Values +------------------------------ + +The Scalar-specific configuration is only available for repos using the +GVFS protocol. + +### Cache Server URL + +When using an enlistment cloned with `scalar clone` and the GVFS protocol, +you will have a value called the cache server URL. Cache servers are a feature +of the GVFS protocol to provide low-latency access to the on-demand object +requests. This modifies the `gvfs.cache-server` setting in your local Git config +file. + +Run `scalar cache-server --get` to see the current cache server. + +Run `scalar cache-server --list` to see the available cache server URLs. + +Run `scalar cache-server --set=` to set your cache server to ``. diff --git a/convert.c b/convert.c index 77f06fcfdba018..729055061c3c50 100644 --- a/convert.c +++ b/convert.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "advice.h" +#include "gvfs.h" #include "config.h" #include "convert.h" #include "copy.h" @@ -563,6 +564,10 @@ static int crlf_to_git(struct index_state *istate, if (!buf) return 1; + if (gvfs_config_is_set(istate && istate->repo ? istate->repo : the_repository, + GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -602,6 +607,9 @@ static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf, if (!will_convert_lf_to_crlf(&stats, crlf_action)) return 0; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -711,6 +719,9 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le struct async async; struct filter_params params; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("Filter \"%s\" not supported when running under GVFS", cmd); + memset(&async, 0, sizeof(async)); async.proc = filter_buffer_or_fd; async.data = ¶ms; @@ -1131,6 +1142,9 @@ static int ident_to_git(const char *src, size_t len, if (!buf) return 1; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -1179,6 +1193,9 @@ static int ident_to_worktree(const char *src, size_t len, if (!cnt) return 0; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -1631,6 +1648,9 @@ static int lf_to_crlf_filter_fn(struct stream_filter *filter, size_t count, o = 0; struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* * We may be holding onto the CR to see if it is followed by a * LF, in which case we would need to go to the main loop. @@ -1875,6 +1895,9 @@ static int ident_filter_fn(struct stream_filter *filter, struct ident_filter *ident = (struct ident_filter *)filter; static const char head[] = "$Id"; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + if (!input) { /* drain upon eof */ switch (ident->state) { diff --git a/credential.c b/credential.c index af964189363b28..7748a0ab26d0d5 100644 --- a/credential.c +++ b/credential.c @@ -258,7 +258,7 @@ static char *credential_ask_one(const char *what, struct credential *c, strbuf_release(&desc); strbuf_release(&prompt); - return xstrdup(r); + return xstrdup(r); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand } static int credential_getpass(struct repository *r, struct credential *c) @@ -461,6 +461,8 @@ static int run_credential_helper(struct credential *c, else helper.no_stdout = 1; + helper.trace2_child_class = "cred"; + if (start_command(&helper) < 0) return -1; diff --git a/date.c b/date.c index 014065b419aee7..8189853343cd77 100644 --- a/date.c +++ b/date.c @@ -524,14 +524,14 @@ static int set_date(int year, int month, int day, struct tm *now_tm, time_t now, if (year == -1) { if (!now_tm) return 1; - r->tm_year = now_tm->tm_year; + r->tm_year = now_tm->tm_year; // CodeQL [SM03231] justification: Git's custom date parser intentionally handles years without leap year validation } else if (year >= 1970 && year < 2100) r->tm_year = year - 1900; else if (year > 70 && year < 100) r->tm_year = year; else if (year < 38) - r->tm_year = year + 100; + r->tm_year = year + 100; // CodeQL [SM03231] justification: Git's date parser handles century offsets without leap year validation by design else return -1; if (!now_tm) @@ -548,7 +548,7 @@ static int set_date(int year, int month, int day, struct tm *now_tm, time_t now, tm->tm_mon = r->tm_mon; tm->tm_mday = r->tm_mday; if (year != -1) - tm->tm_year = r->tm_year; + tm->tm_year = r->tm_year; // CodeQL [SM03231] justification: Git's date parser copies year values without requiring leap year validation return 0; } return -1; @@ -780,11 +780,11 @@ static int match_digit(const char *date, struct tm *tm, int *offset, int *tm_gmt /* Two-digit year? */ if (n == 2 && tm->tm_year < 0) { if (num < 10 && tm->tm_mday >= 0) { - tm->tm_year = num + 100; + tm->tm_year = num + 100; // CodeQL [SM03231] justification: Git's digit parser handles century calculation without leap year validation return n; } if (num >= 70) { - tm->tm_year = num; + tm->tm_year = num; // CodeQL [SM03231] justification: Git's legacy date parser handles two-digit years without leap year validation by design return n; } } @@ -1092,7 +1092,7 @@ static time_t update_tm(struct tm *tm, struct tm *now, time_t sec) if (tm->tm_year < 0) { tm->tm_year = now->tm_year; if (tm->tm_mon > now->tm_mon) - tm->tm_year--; + tm->tm_year--; // CodeQL [SM03231] justification: Git's date parser adjusts year to handle month comparisons without leap year validation } n = mktime(tm) - sec; @@ -1119,9 +1119,9 @@ static void pending_number(struct tm *tm, int *num) if (number > 1969 && number < 2100) tm->tm_year = number - 1900; else if (number > 69 && number < 100) - tm->tm_year = number; + tm->tm_year = number; // CodeQL [SM03231] justification: Git's approxidate parser intentionally assigns years without leap year checks else if (number < 38) - tm->tm_year = 100 + number; + tm->tm_year = 100 + number; // CodeQL [SM03231] justification: Git's approxidate parser handles century calculation without leap year validation /* We screw up for number = 00 ? */ } } @@ -1330,7 +1330,7 @@ static const char *approxidate_alpha(const char *date, struct tm *tm, struct tm *num = 0; while (n < 0) { n += 12; - tm->tm_year--; + tm->tm_year--; // CodeQL [SM03231] justification: Git's approxidate parser adjusts years for month calculations without leap year concerns } tm->tm_mon = n; *touched = 1; @@ -1339,7 +1339,7 @@ static const char *approxidate_alpha(const char *date, struct tm *tm, struct tm if (match_string(date, "years") >= 4) { update_tm(tm, now, 0); /* fill in date fields if needed */ - tm->tm_year -= *num; + tm->tm_year -= *num; // CodeQL [SM03231] justification: Git's approxidate parser subtracts years without leap year validation by design *num = 0; *touched = 1; return end; diff --git a/diagnose.c b/diagnose.c index 5092bf80d35fdd..9d51ded427a265 100644 --- a/diagnose.c +++ b/diagnose.c @@ -12,6 +12,7 @@ #include "parse-options.h" #include "repository.h" #include "write-or-die.h" +#include "config.h" struct archive_dir { const char *path; @@ -71,6 +72,39 @@ static int dir_file_stats(struct odb_source *source, void *data) return 0; } +static void dir_stats(struct strbuf *buf, const char *path) +{ + DIR *dir = opendir(path); + struct dirent *e; + struct stat e_stat; + struct strbuf file_path = STRBUF_INIT; + size_t base_path_len; + + if (!dir) + return; + + strbuf_addstr(buf, "Contents of "); + strbuf_add_absolute_path(buf, path); + strbuf_addstr(buf, ":\n"); + + strbuf_add_absolute_path(&file_path, path); + strbuf_addch(&file_path, '/'); + base_path_len = file_path.len; + + while ((e = readdir(dir)) != NULL) + if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) { + strbuf_setlen(&file_path, base_path_len); + strbuf_addstr(&file_path, e->d_name); + if (!stat(file_path.buf, &e_stat)) + strbuf_addf(buf, "%-70s %16"PRIuMAX"\n", + e->d_name, + (uintmax_t)e_stat.st_size); + } + + strbuf_release(&file_path); + closedir(dir); +} + static int count_files(struct strbuf *path) { DIR *dir = opendir(path->buf); @@ -185,7 +219,8 @@ int create_diagnostics_archive(struct repository *r, struct strvec archiver_args = STRVEC_INIT; char **argv_copy = NULL; int stdout_fd = -1, archiver_fd = -1; - struct strbuf buf = STRBUF_INIT; + char *cache_server_url = NULL, *shared_cache = NULL; + struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; int res; struct archive_dir archive_dirs[] = { { ".git", 0 }, @@ -220,6 +255,13 @@ int create_diagnostics_archive(struct repository *r, get_version_info(&buf, 1); strbuf_addf(&buf, "Repository root: %s\n", r->worktree); + + repo_config_get_string(r, "gvfs.cache-server", &cache_server_url); + repo_config_get_string(r, "gvfs.sharedCache", &shared_cache); + strbuf_addf(&buf, "Cache Server: %s\nLocal Cache: %s\n\n", + cache_server_url ? cache_server_url : "None", + shared_cache ? shared_cache : "None"); + get_disk_info(&buf); write_or_die(stdout_fd, buf.buf, buf.len); strvec_pushf(&archiver_args, @@ -250,6 +292,52 @@ int create_diagnostics_archive(struct repository *r, } } + if (shared_cache) { + size_t path_len; + + strbuf_reset(&buf); + strbuf_addf(&path, "%s/pack", shared_cache); + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=packs-cached.txt:"); + dir_stats(&buf, path.buf); + strvec_push(&archiver_args, buf.buf); + + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=objects-cached.txt:"); + loose_objs_stats(&buf, shared_cache); + strvec_push(&archiver_args, buf.buf); + + strbuf_reset(&path); + strbuf_addf(&path, "%s/info", shared_cache); + path_len = path.len; + + if (is_directory(path.buf)) { + DIR *dir = opendir(path.buf); + struct dirent *e; + + while ((e = readdir(dir))) { + if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name)) + continue; + if (e->d_type == DT_DIR) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "--add-virtual-file=info/%s:", e->d_name); + + strbuf_setlen(&path, path_len); + strbuf_addch(&path, '/'); + strbuf_addstr(&path, e->d_name); + + if (strbuf_read_file(&buf, path.buf, 0) < 0) { + res = error_errno(_("could not read '%s'"), path.buf); + goto diagnose_cleanup; + } + strvec_push(&archiver_args, buf.buf); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand + } + closedir(dir); + } + } + strvec_pushl(&archiver_args, "--prefix=", oid_to_hex(r->hash_algo->empty_tree), "--", NULL); @@ -263,10 +351,13 @@ int create_diagnostics_archive(struct repository *r, goto diagnose_cleanup; } - fprintf(stderr, "\n" - "Diagnostics complete.\n" - "All of the gathered info is captured in '%s'\n", - zip_path->buf); + strbuf_reset(&buf); + strbuf_addf(&buf, "\n" + "Diagnostics complete.\n" + "All of the gathered info is captured in '%s'\n", + zip_path->buf); + write_or_die(stdout_fd, buf.buf, buf.len); + write_or_die(2, buf.buf, buf.len); diagnose_cleanup: if (archiver_fd >= 0) { @@ -277,6 +368,8 @@ int create_diagnostics_archive(struct repository *r, free(argv_copy); strvec_clear(&archiver_args); strbuf_release(&buf); + free(cache_server_url); + free(shared_cache); return res; } diff --git a/diff.c b/diff.c index ab56851b36d658..d27ddd9b18bfd2 100644 --- a/diff.c +++ b/diff.c @@ -56,6 +56,7 @@ static int diff_detect_rename_default; static int diff_indent_heuristic = 1; static int diff_rename_limit_default = 1000; +static int diff_rename_score_default; static int diff_suppress_blank_empty; static enum git_colorbool diff_use_color_default = GIT_COLOR_UNKNOWN; static int diff_color_moved_default; @@ -485,6 +486,16 @@ int git_diff_basic_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "diff.renamethreshold")) { + const char *arg = value; + if (!value) + return config_error_nonbool(var); + diff_rename_score_default = parse_rename_score(&arg); + if (*arg) + return error(_("invalid value for '%s': '%s'"), var, value); + return 0; + } + if (userdiff_config(var, value) < 0) return -1; @@ -4400,6 +4411,13 @@ static int reuse_worktree_file(struct index_state *istate, has_object_pack(istate->repo, oid)) return 0; + /* + * If this path does not match our sparse-checkout definition, + * then the file will not be in the working directory. + */ + if (!path_in_sparse_checkout(name, istate)) + return 0; + /* * Similarly, if we'd have to convert the file contents anyway, that * makes the optimization not worthwhile. @@ -5134,6 +5152,7 @@ void repo_diff_setup(struct repository *r, struct diff_options *options) options->add_remove = diff_addremove; options->use_color = diff_use_color_default; options->detect_rename = diff_detect_rename_default; + options->rename_score = diff_rename_score_default; options->xdl_opts |= diff_algorithm; if (diff_indent_heuristic) DIFF_XDL_SET(options, INDENT_HEURISTIC); diff --git a/dir.c b/dir.c index 2a171080a87ba5..f1703356ee5502 100644 --- a/dir.c +++ b/dir.c @@ -11,6 +11,7 @@ #include "git-compat-util.h" #include "abspath.h" +#include "virtualfilesystem.h" #include "config.h" #include "convert.h" #include "dir.h" @@ -1538,6 +1539,19 @@ enum pattern_match_result path_matches_pattern_list( int result = NOT_MATCHED; size_t slash_pos; + if (core_virtualfilesystem) { + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype == DT_UNKNOWN) + *dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen); + if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0) + return 1; + } + if (!pl->use_cone_patterns) { pattern = last_matching_pattern_from_list(pathname, pathlen, basename, dtype, pl, istate); @@ -1629,6 +1643,13 @@ static int path_in_sparse_checkout_1(const char *path, enum pattern_match_result match = UNDECIDED; const char *end, *slash; + /* + * When using a virtual filesystem, there aren't really patterns + * to follow, but be extra careful to skip this check. + */ + if (core_virtualfilesystem) + return 1; + /* * We default to accepting a path if the path is empty, there are no * patterns, or the patterns are of the wrong type. @@ -1884,8 +1905,22 @@ struct path_pattern *last_matching_pattern(struct dir_struct *dir, int is_excluded(struct dir_struct *dir, struct index_state *istate, const char *pathname, int *dtype_p) { - struct path_pattern *pattern = - last_matching_pattern(dir, istate, pathname, dtype_p); + struct path_pattern *pattern; + + if (core_virtualfilesystem) { + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype_p == DT_UNKNOWN) + *dtype_p = resolve_dtype(DT_UNKNOWN, istate, pathname, strlen(pathname)); + if (is_excluded_from_virtualfilesystem(pathname, strlen(pathname), *dtype_p) > 0) + return 1; + } + + pattern = last_matching_pattern(dir, istate, pathname, dtype_p); if (pattern) return pattern->flags & PATTERN_FLAG_NEGATIVE ? 0 : 1; return 0; @@ -2503,6 +2538,8 @@ static enum path_treatment treat_path(struct dir_struct *dir, repo_ignore_case(the_repository)); if (dtype != DT_DIR && has_path_in_index) return path_none; + if (is_excluded_from_virtualfilesystem(path->buf, path->len, dtype) > 0) + return path_excluded; /* * When we are looking at a directory P in the working tree, @@ -2707,6 +2744,8 @@ static void add_path_to_appropriate_result_list(struct dir_struct *dir, /* add the path to the appropriate result list */ switch (state) { case path_excluded: + if (is_excluded_from_virtualfilesystem(path->buf, path->len, DT_DIR) > 0) + break; if (dir->flags & DIR_SHOW_IGNORED) dir_add_name(dir, istate, path->buf, path->len); else if ((dir->flags & DIR_SHOW_IGNORED_TOO) || @@ -3254,6 +3293,8 @@ static int cmp_icase(char a, char b) { if (a == b) return 0; + if (is_dir_sep(a)) + return is_dir_sep(b) ? 0 : -1; if (repo_ignore_case(the_repository)) return toupper(a) - toupper(b); return a - b; diff --git a/entry.c b/entry.c index 5fd4b08991c6ed..1cb10c97d855c4 100644 --- a/entry.c +++ b/entry.c @@ -468,7 +468,7 @@ static void mark_colliding_entries(const struct checkout *state, ce->ce_flags |= CE_MATCHED; /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(state->istate); + ensure_full_index_unaudited(state->istate); for (size_t i = 0; i < state->istate->cache_nr; i++) { struct cache_entry *dup = state->istate->cache[i]; diff --git a/environment.c b/environment.c index 76ee65e62b4823..4e272d765f431e 100644 --- a/environment.c +++ b/environment.c @@ -35,6 +35,7 @@ #include "quote.h" #include "chdir-notify.h" #include "setup.h" +#include "transport.h" #include "ws.h" #include "write-or-die.h" @@ -57,7 +58,9 @@ char *check_roundtrip_encoding; #define OBJECT_CREATION_MODE OBJECT_CREATION_USES_HARDLINKS #endif int grafts_keep_true_parents; +char *core_virtualfilesystem; unsigned long pack_size_limit_cfg; +int core_virtualize_objects; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 @@ -66,6 +69,9 @@ unsigned long pack_size_limit_cfg; #ifndef PROTECT_NTFS_DEFAULT #define PROTECT_NTFS_DEFAULT 1 #endif +int core_use_gvfs_helper; +char *gvfs_cache_server_url; +struct strbuf gvfs_shared_cache_pathname = STRBUF_INIT; /* * The character that begins a commented line in user-editable file @@ -546,8 +552,17 @@ int git_default_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.usegvfshelper")) { + core_use_gvfs_helper = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.sparsecheckout")) { - cfg->apply_sparse_checkout = git_config_bool(var, value); + /* virtual file system relies on the sparse checkout logic so force it on */ + if (core_virtualfilesystem) + cfg->apply_sparse_checkout = 1; + else + cfg->apply_sparse_checkout = git_config_bool(var, value); return 0; } @@ -673,6 +688,37 @@ static int git_default_push_config(const char *var, const char *value) return 0; } +static int git_default_gvfs_config(const char *var, const char *value) +{ + if (!strcmp(var, "gvfs.cache-server")) { + char *v2 = NULL; + + if (!git_config_string(&v2, var, value) && v2 && *v2) { + free(gvfs_cache_server_url); + gvfs_cache_server_url = transport_anonymize_url(v2); + } + free(v2); + return 0; + } + + if (!strcmp(var, "gvfs.sharedcache") && value && *value) { + strbuf_setlen(&gvfs_shared_cache_pathname, 0); + strbuf_addstr(&gvfs_shared_cache_pathname, value); + if (strbuf_normalize_path(&gvfs_shared_cache_pathname) < 0) { + /* + * Pretend it wasn't set. This will cause us to + * fallback to ".git/objects" effectively. + */ + strbuf_release(&gvfs_shared_cache_pathname); + return 0; + } + strbuf_trim_trailing_dir_sep(&gvfs_shared_cache_pathname); + return 0; + } + + return 0; +} + static int git_default_attr_config(const char *var, const char *value) { if (!strcmp(var, "attr.tree")) { @@ -739,6 +785,9 @@ int git_default_config(const char *var, const char *value, if (starts_with(var, "sparse.")) return git_default_sparse_config(var, value); + if (starts_with(var, "gvfs.")) + return git_default_gvfs_config(var, value); + /* Add other config variables here and to Documentation/config.adoc. */ return 0; } diff --git a/environment.h b/environment.h index e7ec5b0437342d..91b9cf899f4a17 100644 --- a/environment.h +++ b/environment.h @@ -235,6 +235,11 @@ extern int minimum_abbrev, default_abbrev; extern int assume_unchanged; extern unsigned long pack_size_limit_cfg; +extern char *core_virtualfilesystem; +extern int core_use_gvfs_helper; +extern char *gvfs_cache_server_url; +extern struct strbuf gvfs_shared_cache_pathname; + extern int grafts_keep_true_parents; const char *get_log_output_encoding(void); @@ -254,5 +259,6 @@ extern int auto_comment_line_char; extern bool warn_on_auto_comment_char; #endif /* !WITH_BREAKING_CHANGES */ +extern int core_virtualize_objects; # endif /* USE_THE_REPOSITORY_VARIABLE */ #endif /* ENVIRONMENT_H */ diff --git a/fetch-pack.c b/fetch-pack.c index b0af6684a6a6a5..ea97eed0bcdc48 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -158,7 +158,7 @@ static struct commit *deref_without_lazy_fetch(const struct object_id *oid, struct tag *tag = (struct tag *) parse_object(the_repository, oid); - if (!tag->tagged) + if (!tag || !tag->tagged) return NULL; if (mark_tags_complete_and_check_obj_db) tag->object.flags |= COMPLETE; diff --git a/fsck.c b/fsck.c index 94c8651c7dfa28..a9af8446f172c8 100644 --- a/fsck.c +++ b/fsck.c @@ -953,7 +953,7 @@ static int fsck_commit(const struct object_id *oid, { struct object_id tree_oid, parent_oid; unsigned author_count; - int err; + int err = 0; const char *buffer_begin = buffer; const char *buffer_end = buffer + size; const char *p; diff --git a/git.c b/git.c index 41a95435d4218a..c9cd5485f09b2e 100644 --- a/git.c +++ b/git.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "gvfs.h" #include "config.h" #include "environment.h" #include "exec-cmd.h" @@ -17,6 +18,8 @@ #include "shallow.h" #include "trace.h" #include "trace2.h" +#include "dir.h" +#include "hook.h" #define RUN_SETUP (1<<0) #define RUN_SETUP_GENTLY (1<<1) @@ -29,6 +32,8 @@ #define DELAY_PAGER_CONFIG (1<<4) #define NO_PARSEOPT (1<<5) /* parse-options is not used */ #define DEPRECATED (1<<6) +#define BLOCK_ON_GVFS_REPO (1<<7) /* command not allowed in GVFS repos */ +#define BLOCK_ON_VFS_ENABLED (1<<8) /* command not allowed when virtual file system is used */ struct cmd_struct { const char *cmd; @@ -467,6 +472,92 @@ static int handle_alias(struct strvec *args, struct string_list *expanded_aliase return ret; } +/* Runs pre/post-command hook */ +static struct strvec sargv = STRVEC_INIT; +static int run_post_hook = 0; +static int exit_code = -1; + +static int run_pre_command_hook(struct repository *r, const char **argv) +{ + char *lock; + int ret = 0; + int gitdir_was_unset = !r->gitdir; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + + /* + * Ensure the global pre/post command hook is only called for + * the outer command and not when git is called recursively + * or spawns multiple commands (like with the alias command) + */ + lock = getenv("COMMAND_HOOK_LOCK"); + if (lock && !strcmp(lock, "true")) + return 0; + setenv("COMMAND_HOOK_LOCK", "true", 1); + + /* call the hook proc */ + strvec_pushv(&sargv, argv); + strvec_pushf(&sargv, "--git-pid=%"PRIuMAX, (uintmax_t)getpid()); + strvec_pushv(&opt.args, sargv.v); + ret = run_hooks_opt(r, "pre-command", &opt); + + /* + * Hook discovery (build_hook_config_map() in hook.c) calls + * repo_config() to read config-driven hook entries, which + * initializes r->config from whatever sources are available + * at that moment. When we entered with r->gitdir == NULL -- + * the normal case for run_builtin(), which calls us before + * the builtin sets up its repository -- that cache contains + * only system/global config, with no repo-level entries. + * + * Later, when the builtin establishes its gitdir and calls + * repo_config() itself, git_config_check_init() short-circuits + * on the still-initialized cache and never re-reads, leaving + * repo-level config silently invisible. Drop the cache here + * so callers re-read fresh once they know the gitdir. + */ + if (gitdir_was_unset && r->config && r->config->hash_initialized) + repo_config_clear(r); + + if (!ret) + run_post_hook = 1; + return ret; +} + +static int run_post_command_hook(struct repository *r) +{ + char *lock; + int ret = 0; + int saved_errno = errno; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + + /* + * Only run post_command if pre_command succeeded in this process + * and we haven't attempted post_command yet. + */ + if (!run_post_hook) + return 0; + run_post_hook = 0; + + lock = getenv("COMMAND_HOOK_LOCK"); + if (!lock || strcmp(lock, "true")) + return 0; + + strvec_pushv(&opt.args, sargv.v); + strvec_pushf(&opt.args, "--exit_code=%u", exit_code); + ret = run_hooks_opt(r, "post-command", &opt); + + errno = saved_errno; + strvec_clear(&sargv); + strvec_clear(&opt.args); + setenv("COMMAND_HOOK_LOCK", "false", 1); + return ret; +} + +static void post_command_hook_atexit(void) +{ + run_post_command_hook(the_repository); +} + static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct repository *repo) { int status, help; @@ -503,16 +594,27 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct if (!help && p->option & NEED_WORK_TREE) setup_work_tree(the_repository); + if (!help && p->option & BLOCK_ON_GVFS_REPO && gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die("'git %s' is not supported on a GVFS repo", p->cmd); + + if (!help && p->option & BLOCK_ON_VFS_ENABLED && gvfs_config_is_set(repo, GVFS_USE_VIRTUAL_FILESYSTEM)) + die("'git %s' is not supported when using the virtual file system", p->cmd); + + if (run_pre_command_hook(the_repository, argv)) + die("pre-command hook aborted command"); + trace_argv_printf(argv, "trace: built-in: git"); trace2_cmd_name(p->cmd); validate_cache_entries(repo->index); - status = p->fn(argc, argv, prefix, no_repo ? NULL : repo); + exit_code = status = p->fn(argc, argv, prefix, no_repo ? NULL : repo); validate_cache_entries(repo->index); if (status) return status; + run_post_command_hook(the_repository); + /* Somebody closed stdout? */ if (fstat(fileno(stdout), &st)) return 0; @@ -583,7 +685,7 @@ static struct cmd_struct commands[] = { { "for-each-repo", cmd_for_each_repo, RUN_SETUP_GENTLY }, { "format-patch", cmd_format_patch, RUN_SETUP }, { "format-rev", cmd_format_rev, RUN_SETUP }, - { "fsck", cmd_fsck, RUN_SETUP }, + { "fsck", cmd_fsck, RUN_SETUP | BLOCK_ON_VFS_ENABLED }, { "fsck-objects", cmd_fsck, RUN_SETUP }, { "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP }, { "gc", cmd_gc, RUN_SETUP }, @@ -628,7 +730,7 @@ static struct cmd_struct commands[] = { { "pack-refs", cmd_pack_refs, RUN_SETUP }, { "patch-id", cmd_patch_id, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "pickaxe", cmd_blame, RUN_SETUP }, - { "prune", cmd_prune, RUN_SETUP }, + { "prune", cmd_prune, RUN_SETUP | BLOCK_ON_VFS_ENABLED }, { "prune-packed", cmd_prune_packed, RUN_SETUP }, { "pull", cmd_pull, RUN_SETUP | NEED_WORK_TREE }, { "push", cmd_push, RUN_SETUP }, @@ -641,7 +743,7 @@ static struct cmd_struct commands[] = { { "remote", cmd_remote, RUN_SETUP }, { "remote-ext", cmd_remote_ext, NO_PARSEOPT }, { "remote-fd", cmd_remote_fd, NO_PARSEOPT }, - { "repack", cmd_repack, RUN_SETUP }, + { "repack", cmd_repack, RUN_SETUP | BLOCK_ON_VFS_ENABLED }, { "replace", cmd_replace, RUN_SETUP }, { "replay", cmd_replay, RUN_SETUP }, { "repo", cmd_repo, RUN_SETUP }, @@ -663,7 +765,7 @@ static struct cmd_struct commands[] = { { "stash", cmd_stash, RUN_SETUP | NEED_WORK_TREE }, { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE }, { "stripspace", cmd_stripspace }, - { "submodule--helper", cmd_submodule__helper, RUN_SETUP }, + { "submodule--helper", cmd_submodule__helper, RUN_SETUP | BLOCK_ON_GVFS_REPO }, { "survey", cmd_survey, RUN_SETUP }, { "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE }, { "symbolic-ref", cmd_symbolic_ref, RUN_SETUP }, @@ -671,6 +773,7 @@ static struct cmd_struct commands[] = { { "unpack-file", cmd_unpack_file, RUN_SETUP | NO_PARSEOPT }, { "unpack-objects", cmd_unpack_objects, RUN_SETUP | NO_PARSEOPT }, { "update-index", cmd_update_index, RUN_SETUP }, + { "update-microsoft-git", cmd_update_microsoft_git }, { "update-ref", cmd_update_ref, RUN_SETUP }, { "update-server-info", cmd_update_server_info, RUN_SETUP }, { "upload-archive", cmd_upload_archive, NO_PARSEOPT }, @@ -815,13 +918,16 @@ static void execv_dashed_external(const char **argv) */ trace_argv_printf(cmd.args.v, "trace: exec:"); + if (run_pre_command_hook(the_repository, cmd.args.v)) + die("pre-command hook aborted command"); + /* * If we fail because the command is not found, it is * OK to return. Otherwise, we just pass along the status code, * or our usual generic code if we were not even able to exec * the program. */ - status = run_command(&cmd); + exit_code = status = run_command(&cmd); /* * If the child process ran and we are now going to exit, emit a @@ -832,6 +938,8 @@ static void execv_dashed_external(const char **argv) exit(status); else if (errno != ENOENT) exit(128); + + run_post_command_hook(the_repository); } static int is_deprecated_command(const char *cmd) @@ -936,6 +1044,7 @@ int cmd_main(int argc, const char **argv) } trace_command_performance(argv); + atexit(post_command_hook_atexit); /* * "git-xxxx" is the same as "git xxxx", but we obviously: @@ -963,10 +1072,14 @@ int cmd_main(int argc, const char **argv) if (!argc) { /* The user didn't specify a command; give them help */ commit_pager_choice(); + if (run_pre_command_hook(the_repository, argv)) + die("pre-command hook aborted command"); printf(_("usage: %s\n\n"), git_usage_string); list_common_cmds_help(); printf("\n%s\n", _(git_more_info_string)); - exit(1); + exit_code = 1; + run_post_command_hook(the_repository); + exit(exit_code); } if (!strcmp("--version", argv[0]) || !strcmp("-v", argv[0])) diff --git a/gvfs-helper-client.c b/gvfs-helper-client.c new file mode 100644 index 00000000000000..79731321386944 --- /dev/null +++ b/gvfs-helper-client.c @@ -0,0 +1,587 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "config.h" +#include "dir.h" +#include "environment.h" +#include "gvfs-helper-client.h" +#include "hex.h" +#include "object-file.h" +#include "object.h" +#include "oidset.h" +#include "packfile.h" +#include "pkt-line.h" +#include "quote.h" +#include "sigchain.h" +#include "strvec.h" +#include "sub-process.h" +#include "trace2.h" + +static struct oidset gh_client__oidset_queued = OIDSET_INIT; +static unsigned long gh_client__oidset_count; + +struct gh_server__process { + struct subprocess_entry subprocess; /* must be first */ + unsigned int supported_capabilities; +}; + +static int gh_server__subprocess_map_initialized; +static struct hashmap gh_server__subprocess_map; +static struct odb_source *gh_client__chosen_odb; + +/* + * The "objects" capability has verbs: "get" and "post" and "prefetch". + */ +#define CAP_OBJECTS (1u<<1) +#define CAP_OBJECTS_NAME "objects" + +#define CAP_OBJECTS__VERB_GET1_NAME "get" +#define CAP_OBJECTS__VERB_POST_NAME "post" +#define CAP_OBJECTS__VERB_PREFETCH_NAME "prefetch" + +static int gh_client__start_fn(struct subprocess_entry *subprocess) +{ + static int versions[] = {1, 0}; + static struct subprocess_capability capabilities[] = { + { CAP_OBJECTS_NAME, CAP_OBJECTS }, + { NULL, 0 } + }; + + struct gh_server__process *entry = (struct gh_server__process *)subprocess; + + return subprocess_handshake(subprocess, "gvfs-helper", versions, + NULL, capabilities, + &entry->supported_capabilities); +} + +/* + * Send the queued OIDs in the OIDSET to gvfs-helper for it to + * fetch from the cache-server or main Git server using "/gvfs/objects" + * POST semantics. + * + * objects.post LF + * ( LF)* + * + * + */ +static int gh_client__send__objects_post(struct child_process *process) +{ + struct oidset_iter iter; + struct object_id *oid; + int err; + + /* + * We assume that all of the packet_ routines call error() + * so that we don't have to. + */ + + err = packet_write_fmt_gently( + process->in, + (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_POST_NAME "\n")); + if (err) + return err; + + oidset_iter_init(&gh_client__oidset_queued, &iter); + while ((oid = oidset_iter_next(&iter))) { + err = packet_write_fmt_gently(process->in, "%s\n", + oid_to_hex(oid)); + if (err) + return err; + } + + err = packet_flush_gently(process->in); + if (err) + return err; + + return 0; +} + +/* + * Send the given OID to gvfs-helper for it to fetch from the + * cache-server or main Git server using "/gvfs/objects" GET + * semantics. + * + * This ignores any queued OIDs. + * + * objects.get LF + * LF + * + * + */ +static int gh_client__send__objects_get(struct child_process *process, + const struct object_id *oid) +{ + int err; + + /* + * We assume that all of the packet_ routines call error() + * so that we don't have to. + */ + + err = packet_write_fmt_gently( + process->in, + (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_GET1_NAME "\n")); + if (err) + return err; + + err = packet_write_fmt_gently(process->in, "%s\n", + oid_to_hex(oid)); + if (err) + return err; + + err = packet_flush_gently(process->in); + if (err) + return err; + + return 0; +} + +/* + * Send a request to gvfs-helper to prefetch packfiles from either the + * cache-server or the main Git server using "/gvfs/prefetch". + * + * objects.prefetch LF + * [ LF] + * + */ +static int gh_client__send__objects_prefetch(struct child_process *process, + timestamp_t seconds_since_epoch) +{ + int err; + + /* + * We assume that all of the packet_ routines call error() + * so that we don't have to. + */ + + err = packet_write_fmt_gently( + process->in, + (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_PREFETCH_NAME "\n")); + if (err) + return err; + + if (seconds_since_epoch) { + err = packet_write_fmt_gently(process->in, "%" PRItime "\n", + seconds_since_epoch); + if (err) + return err; + } + + err = packet_flush_gently(process->in); + if (err) + return err; + + return 0; +} + +/* + * Update the loose object cache to include the newly created + * object. + */ +static void gh_client__update_loose_cache(const char *line) +{ + const char *v1_oid; + struct object_id oid; + + if (!skip_prefix(line, "loose ", &v1_oid)) + BUG("update_loose_cache: invalid line '%s'", line); + + if (get_oid_hex(v1_oid, &oid)) + BUG("update_loose_cache: invalid line '%s'", line); + + odb_source_loose_cache_add_new_oid(gh_client__chosen_odb, &oid); +} + +/* + * CAP_OBJECTS verbs return the same format response: + * + * + * * + * + * + * + * Where: + * + * ::= odb SP LF + * + * ::= / + * + * ::= packfile SP LF + * + * ::= loose SP LF + * + * ::= ok LF + * / partial LF + * / error SP LF + * + * Note that `gvfs-helper` controls how/if it chunks the request when + * it talks to the cache-server and/or main Git server. So it is + * possible for us to receive many packfiles and/or loose objects *AND + * THEN* get a hard network error or a 404 on an individual object. + * + * If we get a partial result, we can let the caller try to continue + * -- for example, maybe an immediate request for a tree object was + * grouped with a queued request for a blob. The tree-walk *might* be + * able to continue and let the 404 blob be handled later. + */ +static int gh_client__objects__receive_response( + struct child_process *process, + enum gh_client__created *p_ghc, + int *p_nr_loose, int *p_nr_packfile) +{ + enum gh_client__created ghc = GHC__CREATED__NOTHING; + const char *v1; + char *line; + int len; + int nr_loose = 0; + int nr_packfile = 0; + int err = 0; + + while (1) { + /* + * Warning: packet_read_line_gently() calls die() + * despite the _gently moniker. + */ + len = packet_read_line_gently(process->out, NULL, &line); + if ((len < 0) || !line) + break; + + if (starts_with(line, "odb")) { + /* trust that this matches what we expect */ + } + + else if (starts_with(line, "packfile")) { + ghc |= GHC__CREATED__PACKFILE; + nr_packfile++; + } + + else if (starts_with(line, "loose")) { + gh_client__update_loose_cache(line); + ghc |= GHC__CREATED__LOOSE; + nr_loose++; + } + + else if (starts_with(line, "ok")) + ; + else if (starts_with(line, "partial")) + ; + else if (skip_prefix(line, "error ", &v1)) { + error("gvfs-helper error: '%s'", v1); + err = -1; + } + } + + if (ghc & GHC__CREATED__PACKFILE) { + struct odb_source_files *files = odb_source_files_downcast(gh_client__chosen_odb); + odb_source_prepare(&files->packed->base, ODB_PREPARE_FLUSH_CACHES); + } + + *p_ghc = ghc; + *p_nr_loose = nr_loose; + *p_nr_packfile = nr_packfile; + + return err; +} + +/* + * Select the preferred ODB for fetching missing objects. + * This should be the alternate with the same directory + * name as set in `gvfs.sharedCache`. + * + * Fallback to .git/objects if necessary. + */ +static void gh_client__choose_odb(void) +{ + struct odb_source *odb; + + if (gh_client__chosen_odb) + return; + + odb_prepare(the_repository->objects, ODB_PREPARE_FLUSH_CACHES); + gh_client__chosen_odb = the_repository->objects->sources; + + if (!gvfs_shared_cache_pathname.len) + return; + + for (odb = the_repository->objects->sources->next; odb; odb = odb->next) { + if (!fspathcmp(odb->path, gvfs_shared_cache_pathname.buf)) { + gh_client__chosen_odb = odb; + return; + } + } +} + +/* + * Custom exit handler for the `gvfs-helper` subprocesses. + * + * These helper subprocesses will keep waiting for input until they are + * stopped. The default `subprocess_exit_handler()` will instead wait for + * the subprocess to exit, which is not what we want: In case of a fatal + * error, the Git process will exit and the `gvfs-helper` subprocesses will + * need to be stopped explicitly. + * + * The default behavior of `cleanup_children()` does, however, terminate + * the process after calling the `clean_on_exit_handler`. So that's exactly + * what we do here: reproduce the exact same code as + * `subprocess_exit_handler()` modulo waiting for the process that won't + * ever terminate on its own. + */ +static void gh_client__subprocess_exit_handler(struct child_process *process) +{ + sigchain_push(SIGPIPE, SIG_IGN); + /* Closing the pipe signals the subprocess to initiate a shutdown. */ + close(process->in); + close(process->out); + sigchain_pop(SIGPIPE); + /* + * In contrast to subprocess_exit_handler(), do _not_ wait for the + * process to finish on its own accord: It needs to be terminated via + * a signal, which is what `cleanup_children()` will do after this + * function returns. + */ +} + +static struct gh_server__process *gh_client__find_long_running_process( + unsigned int cap_needed) +{ + struct gh_server__process *entry; + struct strvec argv = STRVEC_INIT; + struct strbuf quoted = STRBUF_INIT; + int fallback; + + gh_client__choose_odb(); + + /* + * TODO decide what defaults we want. + */ + strvec_push(&argv, "gvfs-helper"); + strvec_push(&argv, "--cache-server=trust"); + strvec_pushf(&argv, "--shared-cache=%s", + gh_client__chosen_odb->path); + + /* If gvfs.fallback=false, then don't add --fallback. */ + if (!repo_config_get_bool(the_repository, "gvfs.fallback", &fallback) && + !fallback) + strvec_push(&argv, "--no-fallback"); + else + strvec_push(&argv, "--fallback"); + + strvec_push(&argv, "server"); + + sq_quote_argv_pretty("ed, argv.v); + + /* + * Find an existing long-running process with the above command + * line -or- create a new long-running process for this and + * subsequent requests. + */ + if (!gh_server__subprocess_map_initialized) { + gh_server__subprocess_map_initialized = 1; + hashmap_init(&gh_server__subprocess_map, + (hashmap_cmp_fn)cmd2process_cmp, NULL, 0); + entry = NULL; + } else + entry = (struct gh_server__process *)subprocess_find_entry( + &gh_server__subprocess_map, quoted.buf); + + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->supported_capabilities = 0; + + if (subprocess_start_strvec(&gh_server__subprocess_map, + &entry->subprocess, 1, + &argv, gh_client__start_fn)) + FREE_AND_NULL(entry); + else + entry->subprocess.process.clean_on_exit_handler = + gh_client__subprocess_exit_handler; + } + + if (entry && + (entry->supported_capabilities & cap_needed) != cap_needed) { + error("gvfs-helper: does not support needed capabilities"); + subprocess_stop(&gh_server__subprocess_map, + (struct subprocess_entry *)entry); + FREE_AND_NULL(entry); + } + + strvec_clear(&argv); + strbuf_release("ed); + + return entry; +} + +void gh_client__queue_oid(const struct object_id *oid) +{ + /* + * Keep this trace as a printf only, so that it goes to the + * perf log, but not the event log. It is useful for interactive + * debugging, but generates way too much (unuseful) noise for the + * database. + */ + if (trace2_is_enabled()) + trace2_printf("gh_client__queue_oid: %s", oid_to_hex(oid)); + + if (!oidset_insert(&gh_client__oidset_queued, oid)) + gh_client__oidset_count++; +} + +/* + * This routine should actually take a "const struct oid_array *" + * rather than the component parts, but fetch_objects() uses + * this model (because of the call in sha1-file.c). + */ +void gh_client__queue_oid_array(const struct object_id *oids, int oid_nr) +{ + int k; + + for (k = 0; k < oid_nr; k++) + gh_client__queue_oid(&oids[k]); +} + +/* + * Bulk fetch all of the queued OIDs in the OIDSET. + */ +int gh_client__drain_queue(enum gh_client__created *p_ghc) +{ + struct gh_server__process *entry; + struct child_process *process; + int nr_loose = 0; + int nr_packfile = 0; + int err = 0; + + *p_ghc = GHC__CREATED__NOTHING; + + if (!gh_client__oidset_count) + return 0; + + entry = gh_client__find_long_running_process(CAP_OBJECTS); + if (!entry) + return -1; + + trace2_region_enter("gh-client", "objects/post", the_repository); + + process = &entry->subprocess.process; + + sigchain_push(SIGPIPE, SIG_IGN); + + err = gh_client__send__objects_post(process); + if (!err) + err = gh_client__objects__receive_response( + process, p_ghc, &nr_loose, &nr_packfile); + + sigchain_pop(SIGPIPE); + + if (err) { + subprocess_stop(&gh_server__subprocess_map, + (struct subprocess_entry *)entry); + FREE_AND_NULL(entry); + } + + trace2_data_intmax("gh-client", the_repository, + "objects/post/nr_objects", gh_client__oidset_count); + trace2_region_leave("gh-client", "objects/post", the_repository); + + oidset_clear(&gh_client__oidset_queued); + gh_client__oidset_count = 0; + + return err; +} + +/* + * Get exactly 1 object immediately. + * Ignore any queued objects. + */ +int gh_client__get_immediate(const struct object_id *oid, + enum gh_client__created *p_ghc) +{ + struct gh_server__process *entry; + struct child_process *process; + int nr_loose = 0; + int nr_packfile = 0; + int err = 0; + + /* + * Keep this trace as a printf only, so that it goes to the + * perf log, but not the event log. It is useful for interactive + * debugging, but generates way too much (unuseful) noise for the + * database. + */ + if (trace2_is_enabled()) + trace2_printf("gh_client__get_immediate: %s", oid_to_hex(oid)); + + entry = gh_client__find_long_running_process(CAP_OBJECTS); + if (!entry) + return -1; + + trace2_region_enter("gh-client", "objects/get", the_repository); + + process = &entry->subprocess.process; + + sigchain_push(SIGPIPE, SIG_IGN); + + err = gh_client__send__objects_get(process, oid); + if (!err) + err = gh_client__objects__receive_response( + process, p_ghc, &nr_loose, &nr_packfile); + + sigchain_pop(SIGPIPE); + + if (err) { + subprocess_stop(&gh_server__subprocess_map, + (struct subprocess_entry *)entry); + FREE_AND_NULL(entry); + } + + trace2_region_leave("gh-client", "objects/get", the_repository); + + return err; +} + +/* + * Ask gvfs-helper to prefetch commits-and-trees packfiles since a + * given timestamp. + * + * We ignore seconds_since_epoch and use the value from the ODB. + */ +int gh_client__prefetch(timestamp_t seconds_since_epoch UNUSED, + int *nr_packfiles_received) +{ + struct gh_server__process *entry; + struct child_process *process; + enum gh_client__created ghc; + int nr_loose = 0; + int nr_packfile = 0; + int err = 0; + + entry = gh_client__find_long_running_process(CAP_OBJECTS); + if (!entry) + return -1; + + trace2_region_enter("gh-client", "objects/prefetch", the_repository); + + process = &entry->subprocess.process; + + sigchain_push(SIGPIPE, SIG_IGN); + + err = gh_client__send__objects_prefetch(process, /* seconds unknown */ 0); + if (!err) + err = gh_client__objects__receive_response( + process, &ghc, &nr_loose, &nr_packfile); + + sigchain_pop(SIGPIPE); + + if (err) { + subprocess_stop(&gh_server__subprocess_map, + (struct subprocess_entry *)entry); + FREE_AND_NULL(entry); + } + + trace2_data_intmax("gh-client", the_repository, + "prefetch/packfile_count", nr_packfile); + trace2_region_leave("gh-client", "objects/prefetch", the_repository); + + if (nr_packfiles_received) + *nr_packfiles_received = nr_packfile; + + return err; +} diff --git a/gvfs-helper-client.h b/gvfs-helper-client.h new file mode 100644 index 00000000000000..1b436c89cb388b --- /dev/null +++ b/gvfs-helper-client.h @@ -0,0 +1,87 @@ +#ifndef GVFS_HELPER_CLIENT_H +#define GVFS_HELPER_CLIENT_H + +struct repository; +struct commit; +struct object_id; + +enum gh_client__created { + /* + * The _get_ operation did not create anything. If doesn't + * matter if `gvfs-helper` had errors or not -- just that + * nothing was created. + */ + GHC__CREATED__NOTHING = 0, + + /* + * The _get_ operation created one or more packfiles. + */ + GHC__CREATED__PACKFILE = 1<<1, + + /* + * The _get_ operation created one or more loose objects. + * (Not necessarily the for the individual OID you requested.) + */ + GHC__CREATED__LOOSE = 1<<2, + + /* + * The _get_ operation created one or more packfilea *and* + * one or more loose objects. + */ + GHC__CREATED__PACKFILE_AND_LOOSE = (GHC__CREATED__PACKFILE | + GHC__CREATED__LOOSE), +}; + +/* + * Ask `gvfs-helper server` to immediately fetch a single object + * using "/gvfs/objects" GET semantics. + * + * A long-running background process is used to make subsequent + * requests more efficient. + * + * A loose object will be created in the shared-cache ODB and + * in-memory cache updated. + */ +int gh_client__get_immediate(const struct object_id *oid, + enum gh_client__created *p_ghc); + +/* + * Queue this OID for a future fetch using `gvfs-helper service`. + * It does not wait. + * + * Callers should not rely on the queued object being on disk until + * the queue has been drained. + */ +void gh_client__queue_oid(const struct object_id *oid); +void gh_client__queue_oid_array(const struct object_id *oids, int oid_nr); + +/* + * Ask `gvfs-helper server` to fetch the set of queued OIDs using + * "/gvfs/objects" POST semantics. + * + * A long-running background process is used to subsequent requests + * more efficient. + * + * One or more packfiles will be created in the shared-cache ODB. + */ +int gh_client__drain_queue(enum gh_client__created *p_ghc); + +/* + * Ask `gvfs-helper server` to fetch any "prefetch packs" + * available on the server more recent than the requested time. + * + * seconds_since_epoch is ignored. the gvfs-helper will scan the ODB for + * the last received prefetch and ask for ones newer than that. + * + * A long-running background process is used to subsequent requests + * (either prefetch or regular immediate/queued requests) more efficient. + * + * One or more packfiles will be created in the shared-cache ODB. + * + * Returns 0 on success, -1 on error. Optionally also returns the + * number of prefetch packs received. + */ +int gh_client__prefetch(timestamp_t seconds_since_epoch, + int *nr_packfiles_received); + +#endif /* GVFS_HELPER_CLIENT_H */ diff --git a/gvfs-helper.c b/gvfs-helper.c new file mode 100644 index 00000000000000..abb68914b7b370 --- /dev/null +++ b/gvfs-helper.c @@ -0,0 +1,5853 @@ +// TODO Write a man page. Here are some notes for dogfooding. +// TODO +// +// Usage: git gvfs-helper [] [] +// +// : +// +// --remote= // defaults to "origin" +// +// --fallback // boolean. defaults to off +// +// When a fetch from the cache-server fails, automatically +// fallback to the main Git server. This option has no effect +// if no cache-server is defined. +// +// --cache-server= // defaults to "verify" +// +// verify := lookup the set of defined cache-servers using +// "gvfs/config" and confirm that the selected +// cache-server is well-known. Silently disable the +// cache-server if not. (See security notes later.) +// +// error := verify cache-server and abort if not well-known. +// +// trust := do not verify cache-server. just use it, if set. +// +// disable := disable the cache-server and always use the main +// Git server. +// +// --shared-cache= +// +// A relative or absolute pathname to the ODB directory to store +// fetched objects. +// +// If this option is not specified, we default to the value +// in the "gvfs.sharedcache" config setting and then to the +// local ".git/objects" directory. +// +// : +// +// config +// +// Fetch the "gvfs/config" string from the main Git server. +// (The cache-server setting is ignored because cache-servers +// do not support this REST API.) +// +// get +// +// Fetch 1 or more objects one at a time using a "/gvfs/objects" +// GET request. +// +// If a cache-server is configured, +// try it first. Optionally fallback to the main Git server. +// +// The set of objects is given on stdin and is assumed to be +// a list of , one per line. +// +// : +// +// --max-retries= // defaults to "6" +// +// Number of retries after transient network errors. +// Set to zero to disable such retries. +// +// post +// +// Fetch 1 or more objects in bulk using a "/gvfs/objects" POST +// request. +// +// If a cache-server is configured, +// try it first. Optionally fallback to the main Git server. +// +// The set of objects is given on stdin and is assumed to be +// a list of , one per line. +// +// : +// +// --block-size= // defaults to "4000" +// +// Request objects from server in batches of at +// most n objects (not bytes). +// +// --depth= // defaults to "1" +// +// --max-retries= // defaults to "6" +// +// Number of retries after transient network errors. +// Set to zero to disable such retries. +// +// prefetch +// +// Use "/gvfs/prefetch" REST API to fetch 1 or more commits-and-trees +// prefetch packs from the server. +// +// : +// +// --since= // defaults to "0" +// +// Time in seconds since the epoch. If omitted or +// zero, the timestamp from the newest prefetch +// packfile found in the shared-cache ODB is used. +// (This is based upon the packfile name, not the +// mtime.) +// +// The GVFS Protocol defines this value as a way to +// request cached packfiles NEWER THAN this timestamp. +// +// --max-retries= // defaults to "6" +// +// Number of retries after transient network errors. +// Set to zero to disable such retries. +// +// server +// +// Interactive/sub-process mode. Listen for a series of commands +// and data on stdin and return results on stdout. This command +// uses pkt-line format [1] and implements the long-running process +// protocol [2] to communicate with the foreground/parent process. +// +// : +// +// --block-size= // defaults to "4000" +// +// Request objects from server in batches of at +// most n objects (not bytes) when using POST +// requests. +// +// --depth= // defaults to "1" +// +// --max-retries= // defaults to "6" +// +// Number of retries after transient network errors. +// Set to zero to disable such retries. +// +// Interactive verb: objects.get +// +// Fetch 1 or more objects, one at a time, using a +// "/gvfs/objects" GET requests. +// +// Each object will be created as a loose object in the ODB. +// +// Create 1 or more loose objects in the shared-cache ODB. +// (The pathname of the selected ODB is reported at the +// beginning of the response; this should match the pathname +// given on the command line). +// +// git> objects.get +// git> +// git> +// git> ... +// git> +// git> 0000 +// +// git< odb +// git< loose +// git< loose +// git< ... +// git< loose +// git< ok | partial | error +// git< 0000 +// +// Interactive verb: objects.post +// +// Fetch 1 or more objects, in bulk, using one or more +// "/gvfs/objects" POST requests. +// +// Create 1 or more loose objects and/or packfiles in the +// shared-cache ODB. A POST is allowed to respond with +// either loose or packed objects. +// +// git> objects.post +// git> +// git> +// git> ... +// git> +// git> 0000 +// +// git< odb +// git< loose | packfile +// git< loose | packfile +// git< ... +// git< loose | packfile +// git< ok | partial | error +// git< 0000 +// +// Interactive verb: object.prefetch +// +// Fetch 1 or more prefetch packs using a "/gvfs/prefetch" +// request. +// +// git> objects.prefetch +// git> // optional +// git> 0000 +// +// git< odb +// git< packfile +// git< packfile +// git< ... +// git< packfile +// git< ok | error +// git< 0000 +// +// If a cache-server is configured, try it first. +// Optionally fallback to the main Git server. +// +// [1] Documentation/technical/protocol-common.txt +// [2] Documentation/technical/long-running-process-protocol.txt +// [3] See GIT_TRACE_PACKET +// +// endpoint +// +// Fetch the given endpoint from the main Git server (specifying +// `gvfs/config` as endpoint is idempotent to the `config` +// command mentioned above). +// +////////////////////////////////////////////////////////////////// + +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "git-curl-compat.h" +#include "environment.h" +#include "hex.h" +#include "setup.h" +#include "config.h" +#include "remote.h" +#include "connect.h" +#include "strbuf.h" +#include "walker.h" +#include "http.h" +#include "exec-cmd.h" +#include "run-command.h" +#include "pkt-line.h" +#include "string-list.h" +#include "sideband.h" +#include "strvec.h" +#include "credential.h" +#include "oid-array.h" +#include "send-pack.h" +#include "path.h" +#include "protocol.h" +#include "quote.h" +#include "transport.h" +#include "parse-options.h" +#include "odb.h" +#include "object-file.h" +#include "json-writer.h" +#include "tempfile.h" +#include "oidset.h" +#include "dir.h" +#include "url.h" +#include "abspath.h" +#include "progress.h" +#include "trace2.h" +#include "gvfs.h" +#include "trace2/tr2_sid.h" +#include "wrapper.h" +#include "packfile.h" +#include "date.h" +#include "versioncmp.h" +#include "advice.h" +#include "sigchain.h" +#include "thread-utils.h" + +#define TR2_CAT "gvfs-helper" + +static const char * const main_usage[] = { + N_("git gvfs-helper [] config []"), + N_("git gvfs-helper [] get []"), + N_("git gvfs-helper [] post []"), + N_("git gvfs-helper [] prefetch []"), + N_("git gvfs-helper [] server []"), + NULL +}; + +static const char *const objects_get_usage[] = { + N_("git gvfs-helper [] get []"), + NULL +}; + +static const char *const objects_post_usage[] = { + N_("git gvfs-helper [] post []"), + NULL +}; + +static const char *const prefetch_usage[] = { + N_("git gvfs-helper [] prefetch []"), + NULL +}; + +static const char *const server_usage[] = { + N_("git gvfs-helper [] server []"), + NULL +}; + +static const char *const curl_version_usage[] = { + N_("git gvfs-helper [] curl-version [ ]"), + NULL +}; + +/* + * "commitDepth" field in gvfs protocol + */ +#define GH__DEFAULT__OBJECTS_POST__COMMIT_DEPTH 1 + +/* + * Chunk/block size in number of objects we request in each packfile + */ +#define GH__DEFAULT__OBJECTS_POST__BLOCK_SIZE 4000 +#define GH__MIN_OBJECTS_POST__PARALLEL_BLOCK_SIZE 100 + +/* + * Retry attempts (after the initial request) for transient errors and 429s. + */ +#define GH__DEFAULT_MAX_RETRIES 6 + +/* + * Maximum delay in seconds for transient (network) error retries. + */ +#define GH__DEFAULT_MAX_TRANSIENT_BACKOFF_SEC 300 + +/* + * Our exit-codes. + */ +enum gh__error_code { + GH__ERROR_CODE__USAGE = -1, /* will be mapped to usage() */ + GH__ERROR_CODE__OK = 0, + GH__ERROR_CODE__ERROR = 1, /* unspecified */ + GH__ERROR_CODE__CURL_ERROR = 2, + GH__ERROR_CODE__HTTP_401 = 3, + GH__ERROR_CODE__HTTP_404 = 4, + GH__ERROR_CODE__HTTP_429 = 5, + GH__ERROR_CODE__HTTP_503 = 6, + GH__ERROR_CODE__HTTP_OTHER = 7, + GH__ERROR_CODE__UNEXPECTED_CONTENT_TYPE = 8, + + GH__ERROR_CODE__HTTP_ERROR_LIMIT = 9, + + GH__ERROR_CODE__COULD_NOT_CREATE_TEMPFILE = 10, + GH__ERROR_CODE__COULD_NOT_INSTALL_LOOSE = 11, + GH__ERROR_CODE__COULD_NOT_INSTALL_PACKFILE = 12, + GH__ERROR_CODE__SUBPROCESS_SYNTAX = 13, + GH__ERROR_CODE__INDEX_PACK_FAILED = 14, + GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH = 15, +}; + +enum gh__cache_server_mode { + /* verify URL. disable if unknown. */ + GH__CACHE_SERVER_MODE__VERIFY_DISABLE = 0, + /* verify URL. error if unknown. */ + GH__CACHE_SERVER_MODE__VERIFY_ERROR, + /* disable the cache-server, if defined */ + GH__CACHE_SERVER_MODE__DISABLE, + /* trust any cache-server */ + GH__CACHE_SERVER_MODE__TRUST_WITHOUT_VERIFY, +}; + +/* + * The set of command line, config, and environment variables + * that we use as input to decide how we should operate. + */ +static struct gh__cmd_opts { + const char *remote_name; + + int try_fallback; /* to git server if cache-server fails */ + int show_progress; + + int depth; + unsigned int block_size; + int max_retries; + int max_transient_backoff_sec; + + enum gh__cache_server_mode cache_server_mode; +} gh__cmd_opts; + +/* + * The chosen global state derrived from the inputs in gh__cmd_opts. + */ +static struct gh__global { + struct remote *remote; + + struct credential main_creds; + struct credential cache_creds; + + const char *main_url; + char *cache_server_url; + char *cache_server_url_backup; + + struct strbuf buf_odb_path; + + int http_is_initialized; + int cache_server_is_initialized; /* did sub-command look for one */ + int main_creds_need_approval; /* try to only approve them once */ + + unsigned long connect_timeout_ms; + + int prefetch_threads; + int post_threads; +} gh__global; + +enum gh__server_type { + GH__SERVER_TYPE__MAIN = 0, + GH__SERVER_TYPE__CACHE = 1, + + GH__SERVER_TYPE__NR, +}; + +enum gh__verb { + PREFETCH, + GET, + POST, +}; + +static void update_cache_server_for_verb(enum gh__verb verb) +{ + const char *verbstr = NULL; + char *value = NULL; + struct strbuf key = STRBUF_INIT; + + switch (verb) { + case PREFETCH: + verbstr = "prefetch"; + break; + + case GET: + verbstr = "get"; + break; + + case POST: + verbstr = "post"; + break; + + default: + gh__global.cache_server_url_backup = NULL; + return; + } + + gh__global.cache_server_url_backup = gh__global.cache_server_url; + + strbuf_addf(&key, "gvfs.%s.cache-server", verbstr); + + if (!repo_config_get_string(the_repository, key.buf, &value) && + value) { + trace2_data_string("gvfs-helper", the_repository, key.buf, value); + gh__global.cache_server_url = value; + } else { + gh__global.cache_server_url_backup = NULL; + } + + strbuf_release(&key); +} + +static void reset_cache_server(void) +{ + /* + * The backup exists only if the base was replaced with a + * freeable value. + */ + if (gh__global.cache_server_url_backup) { + free(gh__global.cache_server_url); + gh__global.cache_server_url = gh__global.cache_server_url_backup; + gh__global.cache_server_url_backup = NULL; + } +} + +/* + * Build the X-Session-Id header value based on gvfs.sessionkey config. + * + * If gvfs.sessionkey is set, it specifies which config key contains + * a prefix to prepend to the SID. The format is: : + * + * If gvfs.sessionkey is not set or the referenced key doesn't exist, + * the header value is just the SID. + * + * Returns a newly allocated string that must be freed by the caller. + */ +static char *build_session_id_header(void) +{ + struct strbuf header = STRBUF_INIT; + const char *sid = tr2_sid_get(); + char *session_key = NULL; + char *prefix = NULL; + + /* Read gvfs.sessionkey to see if it points to a config key */ + if (!repo_config_get_string(the_repository, "gvfs.sessionkey", &session_key) && + session_key) { + /* Try to read the config key that session_key points to */ + if (!repo_config_get_string(the_repository, session_key, &prefix) && + prefix) { + /* We have a prefix, format as: X-Session-Id: : */ + strbuf_addf(&header, "X-Session-Id: %s:%s", prefix, sid); + free(prefix); + } else { + /* Config key doesn't exist, use just SID */ + strbuf_addf(&header, "X-Session-Id: %s", sid); + } + + free(session_key); + } else { + /* No gvfs.sessionkey configured, use just SID */ + strbuf_addf(&header, "X-Session-Id: %s", sid); + } + + return strbuf_detach(&header, NULL); +} + +static void append_session_id_header(struct curl_slist **headers) +{ + char *session_id_header = build_session_id_header(); + *headers = curl_slist_append(*headers, session_id_header); + free(session_id_header); +} + +static const char *gh__server_type_label[GH__SERVER_TYPE__NR] = { + "(main)", + "(cs)" +}; + +enum gh__objects_mode { + GH__OBJECTS_MODE__NONE = 0, + + /* + * Bulk fetch objects. + * + * But also, force the use of HTTP POST regardless of how many + * objects we are requesting. + * + * The GVFS Protocol treats requests for commit objects + * differently in GET and POST requests WRT whether it + * automatically also fetches the referenced trees. + */ + GH__OBJECTS_MODE__POST, + + /* + * Fetch objects one at a time using HTTP GET. + * + * Force the use of GET (primarily because of the commit + * object treatment). + */ + GH__OBJECTS_MODE__GET, + + /* + * Fetch one or more pre-computed "prefetch packs" containing + * commits and trees. + */ + GH__OBJECTS_MODE__PREFETCH, +}; + +struct gh__azure_throttle +{ + unsigned long tstu_limit; + unsigned long tstu_remaining; + + unsigned long reset_sec; + unsigned long retry_after_sec; +}; + +static void gh__azure_throttle__zero(struct gh__azure_throttle *azure) +{ + azure->tstu_limit = 0; + azure->tstu_remaining = 0; + azure->reset_sec = 0; + azure->retry_after_sec = 0; +} + +#define GH__AZURE_THROTTLE_INIT { \ + .tstu_limit = 0, \ + .tstu_remaining = 0, \ + .reset_sec = 0, \ + .retry_after_sec = 0, \ + } + +static struct gh__azure_throttle gh__global_throttle[GH__SERVER_TYPE__NR] = { + GH__AZURE_THROTTLE_INIT, + GH__AZURE_THROTTLE_INIT, +}; + +/* + * Stolen from http.c + */ +static CURLcode gh__curlinfo_strbuf(CURL *curl, CURLINFO info, struct strbuf *buf) +{ + char *ptr; + CURLcode ret; + + strbuf_reset(buf); + ret = curl_easy_getinfo(curl, info, &ptr); + if (!ret && ptr) + strbuf_addstr(buf, ptr); + return ret; +} + +enum gh__progress_state { + GH__PROGRESS_STATE__START = 0, + GH__PROGRESS_STATE__PHASE1, + GH__PROGRESS_STATE__PHASE2, + GH__PROGRESS_STATE__PHASE3, +}; + +/* + * Parameters to drive an HTTP request (with any necessary retries). + */ +struct gh__request_params { + /* + * b_is_post indicates if the current HTTP request is a POST=1 or + * a GET=0. This is a lower level field used to setup CURL and + * the tempfile used to receive the content. + * + * It is related to, but different from the GH__OBJECTS_MODE__ + * field that we present to the gvfs-helper client or in the CLI + * (which only concerns the semantics of the /gvfs/objects protocol + * on the set of requested OIDs). + * + * For example, we use an HTTP GET to get the /gvfs/config data + * into a buffer. + */ + int b_is_post; + int b_write_to_file; /* write to file=1 or strbuf=0 */ + int b_permit_cache_server_if_defined; + + enum gh__objects_mode objects_mode; + enum gh__server_type server_type; + + int k_attempt; /* robust retry attempt */ + int k_transient_delay_sec; /* delay before transient error retries */ + + unsigned long object_count; /* number of objects being fetched */ + + const struct strbuf *post_payload; /* POST body to send */ + + struct curl_slist *headers; /* additional http headers to send */ + struct tempfile *tempfile; /* for response content when file */ + struct strbuf *buffer; /* for response content when strbuf */ + struct strbuf tr2_label; /* for trace2 regions */ + + struct object_id loose_oid; + + /* + * Note that I am putting all of the progress-related instance data + * inside the request-params in the hope that we can eventually + * do multi-threaded/concurrent HTTP requests when chunking + * large requests. However, the underlying "struct progress" API + * is not thread safe (that is, it doesn't allow concurrent progress + * reports (since that might require multiple lines on the screen + * or something)). + */ + enum gh__progress_state progress_state; + struct strbuf progress_base_phase2_msg; + struct strbuf progress_base_phase3_msg; + + /* + * The buffer for the formatted progress message is shared by the + * "struct progress" API and must remain valid for the duration of + * the start_progress..stop_progress lifespan. + */ + struct strbuf progress_msg; + struct progress *progress; + + struct strbuf e2eid; + + struct string_list *result_list; /* we do not own this */ +}; + +#define GH__REQUEST_PARAMS_INIT { \ + .b_is_post = 0, \ + .b_write_to_file = 0, \ + .b_permit_cache_server_if_defined = 1, \ + .server_type = GH__SERVER_TYPE__MAIN, \ + .k_attempt = 0, \ + .k_transient_delay_sec = 0, \ + .object_count = 0, \ + .post_payload = NULL, \ + .headers = NULL, \ + .tempfile = NULL, \ + .buffer = NULL, \ + .tr2_label = STRBUF_INIT, \ + .loose_oid = {{0}}, \ + .progress_state = GH__PROGRESS_STATE__START, \ + .progress_base_phase2_msg = STRBUF_INIT, \ + .progress_base_phase3_msg = STRBUF_INIT, \ + .progress_msg = STRBUF_INIT, \ + .progress = NULL, \ + .e2eid = STRBUF_INIT, \ + .result_list = NULL, \ + } + +static void gh__request_params__release(struct gh__request_params *params) +{ + if (!params) + return; + + params->post_payload = NULL; /* we do not own this */ + + curl_slist_free_all(params->headers); + params->headers = NULL; + + delete_tempfile(¶ms->tempfile); + + params->buffer = NULL; /* we do not own this */ + + strbuf_release(¶ms->tr2_label); + + strbuf_release(¶ms->progress_base_phase2_msg); + strbuf_release(¶ms->progress_base_phase3_msg); + strbuf_release(¶ms->progress_msg); + + stop_progress(¶ms->progress); + params->progress = NULL; + + strbuf_release(¶ms->e2eid); + + params->result_list = NULL; /* we do not own this */ +} + +/* + * How we handle retries for various unexpected network errors. + */ +enum gh__retry_mode { + /* + * The operation was successful, so no retry is needed. + * Use this for HTTP 200, for example. + */ + GH__RETRY_MODE__SUCCESS = 0, + + /* + * Retry using the normal 401 Auth mechanism. + */ + GH__RETRY_MODE__HTTP_401, + + /* + * Fail because at least one of the requested OIDs does not exist. + */ + GH__RETRY_MODE__FAIL_404, + + /* + * A transient network error, such as dropped connection + * or network IO error. Our belief is that a retry MAY + * succeed. (See Gremlins and Cosmic Rays....) + */ + GH__RETRY_MODE__TRANSIENT, + + /* + * Request was blocked completely because of a 429. + */ + GH__RETRY_MODE__HTTP_429, + + /* + * Request failed because the server was (temporarily?) offline. + */ + GH__RETRY_MODE__HTTP_503, + + /* + * The operation had a hard failure and we have no + * expectation that a second attempt will give a different + * answer, such as a bad hostname or a mal-formed URL. + */ + GH__RETRY_MODE__HARD_FAIL, +}; + +/* + * Bucket to describe the results of an HTTP requests (may be + * overwritten during retries so that it describes the final attempt). + */ +struct gh__response_status { + struct strbuf error_message; + struct strbuf content_type; + enum gh__error_code ec; + enum gh__retry_mode retry; + intmax_t bytes_received; + struct gh__azure_throttle *azure; +}; + +#define GH__RESPONSE_STATUS_INIT { \ + .error_message = STRBUF_INIT, \ + .content_type = STRBUF_INIT, \ + .ec = GH__ERROR_CODE__OK, \ + .retry = GH__RETRY_MODE__SUCCESS, \ + .bytes_received = 0, \ + .azure = NULL, \ + } + +static void gh__response_status__zero(struct gh__response_status *s) +{ + strbuf_setlen(&s->error_message, 0); + strbuf_setlen(&s->content_type, 0); + s->ec = GH__ERROR_CODE__OK; + s->retry = GH__RETRY_MODE__SUCCESS; + s->bytes_received = 0; + s->azure = NULL; +} + +static void install_result(struct gh__request_params *params, + struct gh__response_status *status); + +/* + * Log the E2EID for the current request. + * + * Since every HTTP request to the cache-server and to the main Git server + * will send back a unique E2EID (probably a GUID), we don't want to overload + * telemetry with each ID -- rather, only the ones for which there was a + * problem and that may be helpful in a post mortem. + */ +static void log_e2eid(struct gh__request_params *params, + struct gh__response_status *status) +{ + if (!params->e2eid.len) + return; + + switch (status->retry) { + default: + case GH__RETRY_MODE__SUCCESS: + case GH__RETRY_MODE__HTTP_401: + case GH__RETRY_MODE__FAIL_404: + return; + + case GH__RETRY_MODE__HARD_FAIL: + case GH__RETRY_MODE__TRANSIENT: + case GH__RETRY_MODE__HTTP_429: + case GH__RETRY_MODE__HTTP_503: + break; + } + + if (trace2_is_enabled()) { + struct strbuf key = STRBUF_INIT; + + strbuf_addstr(&key, "e2eid"); + strbuf_addstr(&key, gh__server_type_label[params->server_type]); + + trace2_data_string(TR2_CAT, NULL, key.buf, + params->e2eid.buf); + + strbuf_release(&key); + } +} + +/* + * Normalize a few HTTP response codes before we try to decide + * how to dispatch on them. + */ +static long gh__normalize_odd_codes(struct gh__request_params *params, + long http_response_code) +{ + if (params->server_type == GH__SERVER_TYPE__CACHE && + http_response_code == 400) { + /* + * The cache-server sends a somewhat bogus 400 instead of + * the normal 401 when AUTH is required. Fixup the status + * to hide that. + * + * TODO Technically, the cache-server could send a 400 + * TODO for many reasons, not just for their bogus + * TODO pseudo-401, but we're going to assume it is a + * TODO 401 for now. We should confirm the expected + * TODO error message in the response-body. + */ + return 401; + } + + if (http_response_code == 203) { + /* + * A proxy server transformed a 200 from the origin server + * into a 203. We don't care about the subtle distinction. + */ + return 200; + } + + return http_response_code; +} + +/* + * Map HTTP response codes into a retry strategy. + * See https://en.wikipedia.org/wiki/List_of_HTTP_status_codes + * + * https://docs.microsoft.com/en-us/azure/devops/integrate/concepts/rate-limits?view=azure-devops + */ +static void compute_retry_mode_from_http_response( + struct gh__response_status *status, + long http_response_code) +{ + switch (http_response_code) { + + case 200: + status->retry = GH__RETRY_MODE__SUCCESS; + status->ec = GH__ERROR_CODE__OK; + return; + + case 301: /* all the various flavors of HTTP Redirect */ + case 302: + case 303: + case 304: + case 305: + case 306: + case 307: + case 308: + /* + * TODO Consider a redirected-retry (with or without + * TODO a Retry-After header). + */ + goto hard_fail; + + case 401: + strbuf_addstr(&status->error_message, + "(http:401) Not Authorized"); + status->retry = GH__RETRY_MODE__HTTP_401; + status->ec = GH__ERROR_CODE__HTTP_401; + return; + + case 404: + /* + * TODO if params->object_count > 1, consider + * TODO splitting the request into 2 halves + * TODO and retrying each half in series. + */ + strbuf_addstr(&status->error_message, + "(http:404) Not Found"); + status->retry = GH__RETRY_MODE__FAIL_404; + status->ec = GH__ERROR_CODE__HTTP_404; + return; + + case 429: + /* + * This is a hard block because we've been bad. + */ + strbuf_addstr(&status->error_message, + "(http:429) Too Many Requests [throttled]"); + status->retry = GH__RETRY_MODE__HTTP_429; + status->ec = GH__ERROR_CODE__HTTP_429; + + trace2_data_string(TR2_CAT, NULL, "error/http", + status->error_message.buf); + return; + + case 503: + /* + * We assume that this comes with a "Retry-After" header like 429s. + */ + strbuf_addstr(&status->error_message, + "(http:503) Server Unavailable [throttled]"); + status->retry = GH__RETRY_MODE__HTTP_503; + status->ec = GH__ERROR_CODE__HTTP_503; + + trace2_data_string(TR2_CAT, NULL, "error/http", + status->error_message.buf); + return; + + default: + goto hard_fail; + } + +hard_fail: + strbuf_addf(&status->error_message, "(http:%d) Other [hard_fail]", + (int)http_response_code); + status->retry = GH__RETRY_MODE__HARD_FAIL; + status->ec = GH__ERROR_CODE__HTTP_OTHER; + + trace2_data_string(TR2_CAT, NULL, "error/http", + status->error_message.buf); + return; +} + +/* + * Map CURLE errors code to a retry strategy. + * See and + * https://curl.haxx.se/libcurl/c/libcurl-errors.html + * + * This could be a static table rather than a switch, but + * that is harder to debug and we may want to selectively + * log errors. + * + * I've commented out all of the hard-fail cases for now + * and let the default handle them. This is to indicate + * that I considered them and found them to be not actionable. + * Also, the spelling of some of the CURLE_ symbols seem + * to change between curl releases on different platforms, + * so I'm not going to fight that. + */ +static void compute_retry_mode_from_curl_error( + struct gh__response_status *status, + CURLcode curl_code) +{ + switch (curl_code) { + case CURLE_OK: + status->retry = GH__RETRY_MODE__SUCCESS; + status->ec = GH__ERROR_CODE__OK; + return; + + //se CURLE_UNSUPPORTED_PROTOCOL: goto hard_fail; + //se CURLE_FAILED_INIT: goto hard_fail; + //se CURLE_URL_MALFORMAT: goto hard_fail; + //se CURLE_NOT_BUILT_IN: goto hard_fail; + //se CURLE_COULDNT_RESOLVE_PROXY: goto hard_fail; + //se CURLE_COULDNT_RESOLVE_HOST: goto hard_fail; + case CURLE_COULDNT_CONNECT: goto transient; + //se CURLE_WEIRD_SERVER_REPLY: goto hard_fail; + //se CURLE_REMOTE_ACCESS_DENIED: goto hard_fail; + //se CURLE_FTP_ACCEPT_FAILED: goto hard_fail; + //se CURLE_FTP_WEIRD_PASS_REPLY: goto hard_fail; + //se CURLE_FTP_ACCEPT_TIMEOUT: goto hard_fail; + //se CURLE_FTP_WEIRD_PASV_REPLY: goto hard_fail; + //se CURLE_FTP_WEIRD_227_FORMAT: goto hard_fail; + //se CURLE_FTP_CANT_GET_HOST: goto hard_fail; + case CURLE_HTTP2: goto transient; + //se CURLE_FTP_COULDNT_SET_TYPE: goto hard_fail; + case CURLE_PARTIAL_FILE: goto transient; + //se CURLE_FTP_COULDNT_RETR_FILE: goto hard_fail; + //se CURLE_OBSOLETE20: goto hard_fail; + //se CURLE_QUOTE_ERROR: goto hard_fail; + //se CURLE_HTTP_RETURNED_ERROR: goto hard_fail; + case CURLE_WRITE_ERROR: goto transient; + //se CURLE_OBSOLETE24: goto hard_fail; + case CURLE_UPLOAD_FAILED: goto transient; + //se CURLE_READ_ERROR: goto hard_fail; + //se CURLE_OUT_OF_MEMORY: goto hard_fail; + case CURLE_OPERATION_TIMEDOUT: goto transient; + //se CURLE_OBSOLETE29: goto hard_fail; + //se CURLE_FTP_PORT_FAILED: goto hard_fail; + //se CURLE_FTP_COULDNT_USE_REST: goto hard_fail; + //se CURLE_OBSOLETE32: goto hard_fail; + //se CURLE_RANGE_ERROR: goto hard_fail; + case CURLE_HTTP_POST_ERROR: goto transient; + //se CURLE_SSL_CONNECT_ERROR: goto hard_fail; + //se CURLE_BAD_DOWNLOAD_RESUME: goto hard_fail; + //se CURLE_FILE_COULDNT_READ_FILE: goto hard_fail; + //se CURLE_LDAP_CANNOT_BIND: goto hard_fail; + //se CURLE_LDAP_SEARCH_FAILED: goto hard_fail; + //se CURLE_OBSOLETE40: goto hard_fail; + //se CURLE_FUNCTION_NOT_FOUND: goto hard_fail; + //se CURLE_ABORTED_BY_CALLBACK: goto hard_fail; + //se CURLE_BAD_FUNCTION_ARGUMENT: goto hard_fail; + //se CURLE_OBSOLETE44: goto hard_fail; + //se CURLE_INTERFACE_FAILED: goto hard_fail; + //se CURLE_OBSOLETE46: goto hard_fail; + //se CURLE_TOO_MANY_REDIRECTS: goto hard_fail; + //se CURLE_UNKNOWN_OPTION: goto hard_fail; + //se CURLE_TELNET_OPTION_SYNTAX: goto hard_fail; + //se CURLE_OBSOLETE50: goto hard_fail; + //se CURLE_PEER_FAILED_VERIFICATION: goto hard_fail; + //se CURLE_GOT_NOTHING: goto hard_fail; + //se CURLE_SSL_ENGINE_NOTFOUND: goto hard_fail; + //se CURLE_SSL_ENGINE_SETFAILED: goto hard_fail; + case CURLE_SEND_ERROR: goto transient; + case CURLE_RECV_ERROR: goto transient; + //se CURLE_OBSOLETE57: goto hard_fail; + //se CURLE_SSL_CERTPROBLEM: goto hard_fail; + //se CURLE_SSL_CIPHER: goto hard_fail; + //se CURLE_SSL_CACERT: goto hard_fail; + //se CURLE_BAD_CONTENT_ENCODING: goto hard_fail; + //se CURLE_LDAP_INVALID_URL: goto hard_fail; + //se CURLE_FILESIZE_EXCEEDED: goto hard_fail; + //se CURLE_USE_SSL_FAILED: goto hard_fail; + //se CURLE_SEND_FAIL_REWIND: goto hard_fail; + //se CURLE_SSL_ENGINE_INITFAILED: goto hard_fail; + //se CURLE_LOGIN_DENIED: goto hard_fail; + //se CURLE_TFTP_NOTFOUND: goto hard_fail; + //se CURLE_TFTP_PERM: goto hard_fail; + //se CURLE_REMOTE_DISK_FULL: goto hard_fail; + //se CURLE_TFTP_ILLEGAL: goto hard_fail; + //se CURLE_TFTP_UNKNOWNID: goto hard_fail; + //se CURLE_REMOTE_FILE_EXISTS: goto hard_fail; + //se CURLE_TFTP_NOSUCHUSER: goto hard_fail; + //se CURLE_CONV_FAILED: goto hard_fail; + //se CURLE_CONV_REQD: goto hard_fail; + //se CURLE_SSL_CACERT_BADFILE: goto hard_fail; + //se CURLE_REMOTE_FILE_NOT_FOUND: goto hard_fail; + //se CURLE_SSH: goto hard_fail; + //se CURLE_SSL_SHUTDOWN_FAILED: goto hard_fail; + case CURLE_AGAIN: goto transient; + //se CURLE_SSL_CRL_BADFILE: goto hard_fail; + //se CURLE_SSL_ISSUER_ERROR: goto hard_fail; + //se CURLE_FTP_PRET_FAILED: goto hard_fail; + //se CURLE_RTSP_CSEQ_ERROR: goto hard_fail; + //se CURLE_RTSP_SESSION_ERROR: goto hard_fail; + //se CURLE_FTP_BAD_FILE_LIST: goto hard_fail; + //se CURLE_CHUNK_FAILED: goto hard_fail; + //se CURLE_NO_CONNECTION_AVAILABLE: goto hard_fail; + //se CURLE_SSL_PINNEDPUBKEYNOTMATCH: goto hard_fail; + //se CURLE_SSL_INVALIDCERTSTATUS: goto hard_fail; +#ifdef CURLE_HTTP2_STREAM + case CURLE_HTTP2_STREAM: goto transient; +#endif + default: goto hard_fail; + } + +hard_fail: + strbuf_addf(&status->error_message, "(curl:%d) %s [hard_fail]", + curl_code, curl_easy_strerror(curl_code)); + status->retry = GH__RETRY_MODE__HARD_FAIL; + status->ec = GH__ERROR_CODE__CURL_ERROR; + + trace2_data_string(TR2_CAT, NULL, "error/curl", + status->error_message.buf); + return; + +transient: + strbuf_addf(&status->error_message, "(curl:%d) %s [transient]", + curl_code, curl_easy_strerror(curl_code)); + status->retry = GH__RETRY_MODE__TRANSIENT; + status->ec = GH__ERROR_CODE__CURL_ERROR; + + trace2_data_string(TR2_CAT, NULL, "error/curl", + status->error_message.buf); + return; +} + +/* + * Create a single normalized 'ec' error-code from the status we + * received from the HTTP request. Map a few of the expected HTTP + * status code to 'ec', but don't get too crazy here. + */ +static void gh__response_status__set_from_slot( + struct gh__request_params *params, + struct gh__response_status *status, + const struct active_request_slot *slot) +{ + long http_response_code; + CURLcode curl_code; + + curl_code = slot->results->curl_result; + gh__curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, + &status->content_type); + curl_easy_getinfo(slot->curl, CURLINFO_RESPONSE_CODE, + &http_response_code); + + strbuf_setlen(&status->error_message, 0); + + http_response_code = gh__normalize_odd_codes(params, + http_response_code); + + /* + * Use normalized response/status codes form curl/http to decide + * how to set the error-code we propagate *AND* to decide if we + * we should retry because of transient network problems. + */ + if (curl_code == CURLE_OK || + curl_code == CURLE_HTTP_RETURNED_ERROR) + compute_retry_mode_from_http_response(status, + http_response_code); + else + compute_retry_mode_from_curl_error(status, curl_code); + + if (status->ec != GH__ERROR_CODE__OK) + status->bytes_received = 0; + else if (params->b_write_to_file) + status->bytes_received = (intmax_t)ftell(params->tempfile->fp); + else + status->bytes_received = (intmax_t)params->buffer->len; +} + +static void gh__response_status__release(struct gh__response_status *status) +{ + if (!status) + return; + strbuf_release(&status->error_message); + strbuf_release(&status->content_type); +} + +static int gh__curl_progress_cb(void *clientp, + curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow) +{ + struct gh__request_params *params = clientp; + + /* + * From what I can tell, CURL progress arrives in 3 phases. + * + * [1] An initial connection setup phase where we get [0,0] [0,0]. + * [2] An upload phase where we start sending the request headers + * and body. ulnow will be > 0. ultotal may or may not be 0. + * [3] A download phase where we start receiving the response + * headers and payload body. dlnow will be > 0. dltotal may + * or may not be 0. + * + * If we pass zero for the total to the "struct progress" API, we + * get simple numbers rather than percentages. So our progress + * output format may vary depending. + * + * It is unclear if CURL will give us a final callback after + * everything is finished, so we leave the progress handle open + * and let the caller issue the final stop_progress(). + * + * There is a bit of a mismatch between the CURL API and the + * "struct progress" API. The latter requires us to set the + * progress message when we call one of the start_progress + * methods. We cannot change the progress message while we are + * showing progress state. And we cannot change the denominator + * (total) after we start. CURL may or may not give us the total + * sizes for each phase. + * + * Also be advised that the "struct progress" API eats messages + * so that the screen is only updated every second or so. And + * may not print anything if the start..stop happen in less then + * 2 seconds. Whereas CURL calls this callback very frequently. + * The net-net is that we may not actually see this progress + * message for small/fast HTTP requests. + */ + + switch (params->progress_state) { + case GH__PROGRESS_STATE__START: /* first callback */ + if (dlnow == 0 && ulnow == 0) + goto enter_phase_1; + + if (ulnow) + goto enter_phase_2; + else + goto enter_phase_3; + + case GH__PROGRESS_STATE__PHASE1: + if (dlnow == 0 && ulnow == 0) + return 0; + + if (ulnow) + goto enter_phase_2; + else + goto enter_phase_3; + + case GH__PROGRESS_STATE__PHASE2: + display_progress(params->progress, ulnow); + if (dlnow == 0) + return 0; + + stop_progress(¶ms->progress); + goto enter_phase_3; + + case GH__PROGRESS_STATE__PHASE3: + display_progress(params->progress, dlnow); + return 0; + + default: + return 0; + } + +enter_phase_1: + /* + * Don't bother to create a progress handle during phase [1]. + * Because we get [0,0,0,0], we don't have any data to report + * and would just have to synthesize some type of progress. + * From my testing, phase [1] is fairly quick (probably just + * the SSL handshake), so the "struct progress" API will most + * likely completely eat any messages that we did produce. + */ + params->progress_state = GH__PROGRESS_STATE__PHASE1; + return 0; + +enter_phase_2: + strbuf_setlen(¶ms->progress_msg, 0); + if (params->progress_base_phase2_msg.len) { + if (params->k_attempt > 0) + strbuf_addf(¶ms->progress_msg, "%s [retry %d/%d] (bytes sent)", + params->progress_base_phase2_msg.buf, + params->k_attempt, gh__cmd_opts.max_retries); + else + strbuf_addf(¶ms->progress_msg, "%s (bytes sent)", + params->progress_base_phase2_msg.buf); + params->progress = start_progress(the_repository, params->progress_msg.buf, ultotal); + display_progress(params->progress, ulnow); + } + params->progress_state = GH__PROGRESS_STATE__PHASE2; + return 0; + +enter_phase_3: + strbuf_setlen(¶ms->progress_msg, 0); + if (params->progress_base_phase3_msg.len) { + if (params->k_attempt > 0) + strbuf_addf(¶ms->progress_msg, "%s [retry %d/%d] (bytes received)", + params->progress_base_phase3_msg.buf, + params->k_attempt, gh__cmd_opts.max_retries); + else + strbuf_addf(¶ms->progress_msg, "%s (bytes received)", + params->progress_base_phase3_msg.buf); + params->progress = start_progress(the_repository, params->progress_msg.buf, dltotal); + display_progress(params->progress, dlnow); + } + params->progress_state = GH__PROGRESS_STATE__PHASE3; + return 0; +} + +/* + * Run the request without using "run_one_slot()" because we + * don't want the post-request normalization, error handling, + * and auto-reauth handling in http.c. + */ +static void gh__run_one_slot(struct active_request_slot *slot, + struct gh__request_params *params, + struct gh__response_status *status) +{ + struct strbuf key = STRBUF_INIT; + + strbuf_addbuf(&key, ¶ms->tr2_label); + strbuf_addstr(&key, gh__server_type_label[params->server_type]); + + params->progress_state = GH__PROGRESS_STATE__START; + strbuf_setlen(¶ms->e2eid, 0); + + trace2_region_enter(TR2_CAT, key.buf, NULL); + + if (!start_active_slot(slot)) { + compute_retry_mode_from_curl_error(status, + CURLE_FAILED_INIT); + } else { + run_active_slot(slot); + if (params->b_write_to_file) + fflush(params->tempfile->fp); + + gh__response_status__set_from_slot(params, status, slot); + + log_e2eid(params, status); + + if (status->ec == GH__ERROR_CODE__OK) { + int old_len = key.len; + + /* + * We only log the number of bytes received. + * We do not log the number of objects requested + * because the server may give us more than that + * (such as when we request a commit). + */ + strbuf_addstr(&key, "/nr_bytes"); + trace2_data_intmax(TR2_CAT, NULL, + key.buf, + status->bytes_received); + strbuf_setlen(&key, old_len); + } + } + + if (params->progress) + stop_progress(¶ms->progress); + + if (status->ec == GH__ERROR_CODE__OK && params->b_write_to_file) + install_result(params, status); + + trace2_region_leave(TR2_CAT, key.buf, NULL); + + strbuf_release(&key); +} + +static int option_parse_cache_server_mode(const struct option *opt, + const char *arg, int unset) +{ + if (unset) /* should not happen */ + return error(_("missing value for switch '%s'"), + opt->long_name); + + else if (!strcmp(arg, "verify")) + gh__cmd_opts.cache_server_mode = + GH__CACHE_SERVER_MODE__VERIFY_DISABLE; + + else if (!strcmp(arg, "error")) + gh__cmd_opts.cache_server_mode = + GH__CACHE_SERVER_MODE__VERIFY_ERROR; + + else if (!strcmp(arg, "disable")) + gh__cmd_opts.cache_server_mode = + GH__CACHE_SERVER_MODE__DISABLE; + + else if (!strcmp(arg, "trust")) + gh__cmd_opts.cache_server_mode = + GH__CACHE_SERVER_MODE__TRUST_WITHOUT_VERIFY; + + else + return error(_("invalid value for switch '%s'"), + opt->long_name); + + return 0; +} + +/* + * Let command line args override "gvfs.sharedcache" config setting + * and override the value set by git_default_config(). + * + * The command line is parsed *AFTER* the config is loaded, so + * prepared_alt_odb() has already been called any default or inherited + * shared-cache has already been set. + * + * We have a chance to override it here. + */ +static int option_parse_shared_cache_directory(const struct option *opt, + const char *arg, int unset) +{ + struct strbuf buf_arg = STRBUF_INIT; + + if (unset) /* should not happen */ + return error(_("missing value for switch '%s'"), + opt->long_name); + + strbuf_addstr(&buf_arg, arg); + if (strbuf_normalize_path(&buf_arg) < 0) { + /* + * Pretend command line wasn't given. Use whatever + * settings we already have from the config. + */ + strbuf_release(&buf_arg); + return 0; + } + strbuf_trim_trailing_dir_sep(&buf_arg); + + if (!strbuf_cmp(&buf_arg, &gvfs_shared_cache_pathname)) { + /* + * The command line argument matches what we got from + * the config, so we're already setup correctly. (And + * we have already verified that the directory exists + * on disk.) + */ + strbuf_release(&buf_arg); + return 0; + } + + else if (!gvfs_shared_cache_pathname.len) { + /* + * A shared-cache was requested and we did not inherit one. + * Try it, but let alt_odb_usable() secretly disable it if + * it cannot create the directory on disk. + */ + strbuf_addbuf(&gvfs_shared_cache_pathname, &buf_arg); + + /* Attempt to create the directory, in case it doesn't exist. */ + safe_create_leading_directories(the_repository, + gvfs_shared_cache_pathname.buf); + mkdir(gvfs_shared_cache_pathname.buf, 0777); + + add_gvfs_shared_cache_to_alternates(the_repository->objects, buf_arg.buf); + + strbuf_release(&buf_arg); + return 0; + } else { + /* + * The requested shared-cache is different from the one + * we inherited. Replace the inherited value with this + * one, but smartly fallback if necessary. + */ + struct strbuf buf_prev = STRBUF_INIT; + + strbuf_addbuf(&buf_prev, &gvfs_shared_cache_pathname); + + strbuf_setlen(&gvfs_shared_cache_pathname, 0); + strbuf_addbuf(&gvfs_shared_cache_pathname, &buf_arg); + + /* Attempt to create the directory, in case it doesn't exist. */ + safe_create_leading_directories(the_repository, + gvfs_shared_cache_pathname.buf); + mkdir(gvfs_shared_cache_pathname.buf, 0777); + + add_gvfs_shared_cache_to_alternates(the_repository->objects, buf_arg.buf); + + /* + * alt_odb_usable() releases gvfs_shared_cache_pathname + * if it cannot create the directory on disk, so fallback + * to the previous choice when it fails. + */ + if (!gvfs_shared_cache_pathname.len) + strbuf_addbuf(&gvfs_shared_cache_pathname, + &buf_prev); + + strbuf_release(&buf_arg); + strbuf_release(&buf_prev); + return 0; + } +} + +/* + * Lookup the URL for this remote (defaults to 'origin'). + */ +static void lookup_main_url(void) +{ + /* + * Both VFS and Scalar only work with 'origin', so we expect this. + * The command line arg is mainly for debugging. + */ + if (!gh__cmd_opts.remote_name || !*gh__cmd_opts.remote_name) + gh__cmd_opts.remote_name = "origin"; + + gh__global.remote = remote_get(gh__cmd_opts.remote_name); + if (!gh__global.remote->url.v[0] || !*gh__global.remote->url.v[0]) + die("unknown remote '%s'", gh__cmd_opts.remote_name); + + /* + * Strip out any in-line auth in the origin server URL so that + * we can control which creds we fetch. + * + * Azure DevOps has been known to suggest https URLS of the + * form "https://@dev.azure.com//". + * + * Break that so that we can force the use of a PAT. + */ + gh__global.main_url = transport_anonymize_url(gh__global.remote->url.v[0]); + + trace2_data_string(TR2_CAT, NULL, "remote/url", gh__global.main_url); +} + +static void do__http_get__gvfs_config(struct gh__response_status *status, + struct strbuf *config_data); + +/* + * Find the URL of the cache-server, if we have one. + * + * This routine is called by the initialization code and is allowed + * to call die() rather than returning an 'ec'. + */ +static void select_cache_server(void) +{ + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct strbuf config_data = STRBUF_INIT; + const char *match = NULL; + + /* + * This only indicates that the sub-command actually called + * this routine. We rely on gh__global.cache_server_url to tell + * us if we actually have a cache-server configured. + */ + gh__global.cache_server_is_initialized = 1; + gh__global.cache_server_url = NULL; + + if (gh__cmd_opts.cache_server_mode == GH__CACHE_SERVER_MODE__DISABLE) { + trace2_data_string(TR2_CAT, NULL, "cache/url", "disabled"); + return; + } + + if (!gvfs_cache_server_url || !*gvfs_cache_server_url) { + switch (gh__cmd_opts.cache_server_mode) { + default: + case GH__CACHE_SERVER_MODE__TRUST_WITHOUT_VERIFY: + case GH__CACHE_SERVER_MODE__VERIFY_DISABLE: + trace2_data_string(TR2_CAT, NULL, "cache/url", "unset"); + return; + + case GH__CACHE_SERVER_MODE__VERIFY_ERROR: + die("cache-server not set"); + } + } + + /* + * If the cache-server and main Git server have the same URL, we + * can silently disable the cache-server (by NOT setting the field + * in gh__global and explicitly disable the fallback logic.) + */ + if (!strcmp(gvfs_cache_server_url, gh__global.main_url)) { + gh__cmd_opts.try_fallback = 0; + trace2_data_string(TR2_CAT, NULL, "cache/url", "same"); + return; + } + + if (gh__cmd_opts.cache_server_mode == + GH__CACHE_SERVER_MODE__TRUST_WITHOUT_VERIFY) { + gh__global.cache_server_url = gvfs_cache_server_url; + trace2_data_string(TR2_CAT, NULL, "cache/url", + gvfs_cache_server_url); + return; + } + + /* + * GVFS cache-servers use the main Git server's creds rather + * than having their own creds. This feels like a security + * hole. For example, if the cache-server URL is pointed to a + * bad site, we'll happily send them our creds to the main Git + * server with each request to the cache-server. This would + * allow an attacker to later use our creds to impersonate us + * on the main Git server. + * + * So we optionally verify that the URL to the cache-server is + * well-known by the main Git server. + */ + + do__http_get__gvfs_config(&status, &config_data); + + if (status.ec == GH__ERROR_CODE__OK) { + /* + * The gvfs/config response is in JSON, but I don't think + * we need to parse it and all that. Lets just do a simple + * strstr() and assume it is sufficient. + * + * We do add some context to the pattern to guard against + * some attacks. + */ + struct strbuf pattern = STRBUF_INIT; + + strbuf_addf(&pattern, "\"Url\":\"%s\"", gvfs_cache_server_url); + match = strstr(config_data.buf, pattern.buf); + + strbuf_release(&pattern); + } + + strbuf_release(&config_data); + + if (match) { + gh__global.cache_server_url = gvfs_cache_server_url; + trace2_data_string(TR2_CAT, NULL, "cache/url", + gvfs_cache_server_url); + } + + else if (gh__cmd_opts.cache_server_mode == + GH__CACHE_SERVER_MODE__VERIFY_ERROR) { + if (status.ec != GH__ERROR_CODE__OK) + die("could not verify cache-server '%s': %s", + gvfs_cache_server_url, + status.error_message.buf); + else + die("could not verify cache-server '%s'", + gvfs_cache_server_url); + } + + else if (gh__cmd_opts.cache_server_mode == + GH__CACHE_SERVER_MODE__VERIFY_DISABLE) { + if (status.ec != GH__ERROR_CODE__OK) + warning("could not verify cache-server '%s': %s", + gvfs_cache_server_url, + status.error_message.buf); + else + warning("could not verify cache-server '%s'", + gvfs_cache_server_url); + trace2_data_string(TR2_CAT, NULL, "cache/url", + "disabled"); + } + + gh__response_status__release(&status); +} + +/* + * Read stdin until EOF (or a blank line) and add the desired OIDs + * to the oidset. + * + * Stdin should contain a list of OIDs. Lines may have additional + * text following the OID that we ignore. + */ +static unsigned long read_stdin_for_oids(struct oidset *oids) +{ + struct object_id oid; + struct strbuf buf_stdin = STRBUF_INIT; + unsigned long count = 0; + + do { + if (strbuf_getline(&buf_stdin, stdin) == EOF || !buf_stdin.len) + break; + + if (get_oid_hex(buf_stdin.buf, &oid)) + continue; /* just silently eat it */ + + if (!oidset_insert(oids, &oid)) + count++; + } while (1); + + strbuf_release(&buf_stdin); + return count; +} + +/* + * Build a complete JSON payload for a gvfs/objects POST request + * containing the first `nr_in_block` OIDs found in the OIDSET + * indexed by the given iterator. + * + * https://github.com/microsoft/VFSForGit/blob/master/Protocol.md + * + * Return the number of OIDs we actually put into the payload. + * If only 1 OID was found, also return it. + */ +static unsigned long build_json_payload__gvfs_objects( + struct json_writer *jw_req, + struct oidset_iter *iter, + unsigned long nr_in_block, + struct object_id *oid_out) +{ + unsigned long k; + const struct object_id *oid; + const struct object_id *oid_prev = NULL; + + k = 0; + + jw_init(jw_req); + jw_object_begin(jw_req, 0); + jw_object_intmax(jw_req, "commitDepth", gh__cmd_opts.depth); + jw_object_inline_begin_array(jw_req, "objectIds"); + while (k < nr_in_block && (oid = oidset_iter_next(iter))) { + jw_array_string(jw_req, oid_to_hex(oid)); + k++; + oid_prev = oid; + } + jw_end(jw_req); + jw_end(jw_req); + + if (oid_out) { + if (k == 1) + oidcpy(oid_out, oid_prev); + else + oidclr(oid_out, the_repository->hash_algo); + } + + return k; +} + +/* + * Build a JSON payload for a subset of OIDs from a flat array. + * Used by the parallel POST workers which pre-partition OIDs. + */ +static void build_post_payload(struct json_writer *jw, + const struct object_id * const *oids, + size_t start, size_t count) +{ + size_t k; + char hex[GIT_MAX_HEXSZ + 1]; + + jw_init(jw); + jw_object_begin(jw, 0); + jw_object_intmax(jw, "commitDepth", gh__cmd_opts.depth); + jw_object_inline_begin_array(jw, "objectIds"); + for (k = start; k < start + count; k++) + jw_array_string(jw, oid_to_hex_r(hex, oids[k])); + jw_end(jw); + jw_end(jw); +} + +/* + * Lookup the creds for the main/origin Git server. + */ +static void lookup_main_creds(void) +{ + if (gh__global.main_creds.username && *gh__global.main_creds.username) + return; + + credential_from_url(&gh__global.main_creds, gh__global.main_url); + credential_fill(the_repository, &gh__global.main_creds, 0); + gh__global.main_creds_need_approval = 1; +} + +/* + * If we have a set of creds for the main Git server, tell the credential + * manager to throw them away and ask it to reacquire them. + */ +static void refresh_main_creds(void) +{ + if (gh__global.main_creds.username && *gh__global.main_creds.username) + credential_reject(the_repository, &gh__global.main_creds); + + lookup_main_creds(); + + // TODO should we compare before and after values of u/p and + // TODO shortcut reauth if we already know it will fail? + // TODO if so, return a bool if same/different. +} + +static void approve_main_creds(void) +{ + if (!gh__global.main_creds_need_approval) + return; + + credential_approve(the_repository, &gh__global.main_creds); + gh__global.main_creds_need_approval = 0; +} + +/* + * Build a set of creds for the cache-server based upon the main Git + * server (assuming we have a cache-server configured). + * + * That is, we NEVER fill them directly for the cache-server -- we + * only synthesize them from the filled main creds. + */ +static void synthesize_cache_server_creds(void) +{ + if (!gh__global.cache_server_is_initialized) + BUG("sub-command did not initialize cache-server vars"); + + if (!gh__global.cache_server_url) + return; + + if (gh__global.cache_creds.username && *gh__global.cache_creds.username) + return; + + /* + * Get the main Git server creds so we can borrow the username + * and password when we talk to the cache-server. + */ + lookup_main_creds(); + free(gh__global.cache_creds.username); + gh__global.cache_creds.username = xstrdup(gh__global.main_creds.username); + free(gh__global.cache_creds.password); + gh__global.cache_creds.password = xstrdup(gh__global.main_creds.password); +} + +/* + * Flush and refresh the cache-server creds. Because the cache-server + * does not do 401s (or manage creds), we have to reload the main Git + * server creds first. + * + * That is, we NEVER reject them directly because we never filled them. + */ +static void refresh_cache_server_creds(void) +{ + credential_clear(&gh__global.cache_creds); + + refresh_main_creds(); + synthesize_cache_server_creds(); +} + +/* + * We NEVER approve cache-server creds directly because we never directly + * filled them. However, we should be able to infer that the main ones + * are valid and can approve them if necessary. + */ +static void approve_cache_server_creds(void) +{ + approve_main_creds(); +} + +/* + * Get the pathname to the ODB where we write objects that we download. + */ +static void select_odb(void) +{ + odb_prepare(the_repository->objects, ODB_PREPARE_FLUSH_CACHES); + + strbuf_init(&gh__global.buf_odb_path, 0); + + if (gvfs_shared_cache_pathname.len) + strbuf_addbuf(&gh__global.buf_odb_path, + &gvfs_shared_cache_pathname); + else + strbuf_addstr(&gh__global.buf_odb_path, + repo_get_object_directory(the_repository)); +} + +/* + * Create a unique tempfile or tempfile-pair inside the + * tempPacks directory. + */ +static void my_create_tempfile( + struct gh__response_status *status, + struct repository *repo, + int b_fdopen, + const char *suffix1, struct tempfile **t1, + const char *suffix2, struct tempfile **t2) +{ + static unsigned int nth = 0; + static struct timeval tv = {0}; + static struct tm tm = {0}; + static time_t secs = 0; + static char date[32] = {0}; + + struct strbuf basename = STRBUF_INIT; + struct strbuf buf = STRBUF_INIT; + int len_tp; + enum scld_error scld; + int retries; + + gh__response_status__zero(status); + + if (!nth) { + /* + * Create a unique string to use in the name of all + * tempfiles created by this process. + */ + gettimeofday(&tv, NULL); + secs = tv.tv_sec; + gmtime_r(&secs, &tm); + + xsnprintf(date, sizeof(date), "%4d%02d%02d-%02d%02d%02d-%06ld", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, + (long)tv.tv_usec); + } + + /* + * Create a for this instance/pair using a series + * number . + */ + strbuf_addf(&basename, "t-%s-%04d", date, nth++); + + if (!suffix1 || !*suffix1) + suffix1 = "temp"; + + /* + * Create full pathname as: + * + * "/pack/tempPacks/." + */ + strbuf_setlen(&buf, 0); + strbuf_addbuf(&buf, &gh__global.buf_odb_path); + strbuf_complete(&buf, '/'); + strbuf_addstr(&buf, "pack/tempPacks/"); + len_tp = buf.len; + strbuf_addf( &buf, "%s.%s", basename.buf, suffix1); + + scld = safe_create_leading_directories(the_repository, buf.buf); + if (scld != SCLD_OK && scld != SCLD_EXISTS) { + strbuf_addf(&status->error_message, + "could not create directory for tempfile: '%s'", + buf.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_CREATE_TEMPFILE; + goto cleanup; + } + + retries = 0; + *t1 = repo_create_tempfile(repo, buf.buf); + while (!*t1 && retries < 5) { + retries++; + strbuf_setlen(&buf, len_tp); + strbuf_addf(&buf, "%s-%d.%s", basename.buf, retries, suffix1); + *t1 = repo_create_tempfile(repo, buf.buf); + } + + if (!*t1) { + strbuf_addf(&status->error_message, + "could not create tempfile: '%s'", + buf.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_CREATE_TEMPFILE; + goto cleanup; + } + if (b_fdopen) + fdopen_tempfile(*t1, "w"); + + /* + * Optionally create a peer tempfile with the same basename. + * (This is useful for prefetching .pack and .idx pairs.) + * + * "/pack/tempPacks/." + */ + if (suffix2 && *suffix2 && t2) { + strbuf_setlen(&buf, len_tp); + strbuf_addf( &buf, "%s.%s", basename.buf, suffix2); + + *t2 = repo_create_tempfile(repo, buf.buf); + while (!*t2 && retries < 5) { + retries++; + strbuf_setlen(&buf, len_tp); + strbuf_addf(&buf, "%s-%d.%s", basename.buf, retries, suffix2); + *t2 = repo_create_tempfile(repo, buf.buf); + } + + if (!*t2) { + strbuf_addf(&status->error_message, + "could not create tempfile: '%s'", + buf.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_CREATE_TEMPFILE; + goto cleanup; + } + if (b_fdopen) + fdopen_tempfile(*t2, "w"); + } + +cleanup: + strbuf_release(&buf); + strbuf_release(&basename); +} + +/* + * Create pathnames to the final location of the .pack and .idx + * files in the ODB. These are of the form: + * + * "/pack/-[-]." + * + * For example, for prefetch packs, will be the epoch + * timestamp and will be the packfile hash. + */ +static void create_final_packfile_pathnames( + const char *term_1, const char *term_2, const char *term_3, + struct strbuf *pack_path, struct strbuf *idx_path, + struct strbuf *pack_filename) +{ + struct strbuf base = STRBUF_INIT; + struct strbuf path = STRBUF_INIT; + + if (term_3 && *term_3) + strbuf_addf(&base, "%s-%s-%s", term_1, term_2, term_3); + else + strbuf_addf(&base, "%s-%s", term_1, term_2); + + strbuf_setlen(pack_filename, 0); + strbuf_addf( pack_filename, "%s.pack", base.buf); + + strbuf_addbuf(&path, &gh__global.buf_odb_path); + strbuf_complete(&path, '/'); + strbuf_addstr(&path, "pack/"); + + strbuf_setlen(pack_path, 0); + strbuf_addbuf(pack_path, &path); + strbuf_addf( pack_path, "%s.pack", base.buf); + + strbuf_setlen(idx_path, 0); + strbuf_addbuf(idx_path, &path); + strbuf_addf( idx_path, "%s.idx", base.buf); + + strbuf_release(&base); + strbuf_release(&path); +} + +/* + * Thread-safe packfile finalization: move temp .pack and .idx to + * their final locations. Tolerates races where another thread or + * process installed the same packfile concurrently. + */ +static int my_finalize_packfile_simple(const char *temp_pack, + const char *temp_idx, + const char *final_pack, + const char *final_idx) +{ + if (finalize_object_file_flags(the_repository, temp_pack, final_pack, + FOF_SKIP_COLLISION_CHECK) || + finalize_object_file_flags(the_repository, temp_idx, final_idx, + FOF_SKIP_COLLISION_CHECK)) { + unlink(temp_pack); + unlink(temp_idx); + + if (file_exists(final_pack) && file_exists(final_idx)) { + trace2_printf("%s: assuming ok for %s", + TR2_CAT, final_pack); + return 0; + } + + return -1; + } + + return 0; +} + +/* + * Create a pathname to the loose object in the shared-cache ODB + * with the given OID. Try to "mkdir -p" to ensure the parent + * directories exist. + */ +static int create_loose_pathname_in_odb(struct strbuf *buf_path, + const struct object_id *oid) +{ + enum scld_error scld; + const char *hex; + + hex = oid_to_hex(oid); + + strbuf_setlen(buf_path, 0); + strbuf_addbuf(buf_path, &gh__global.buf_odb_path); + strbuf_complete(buf_path, '/'); + strbuf_add(buf_path, hex, 2); + strbuf_addch(buf_path, '/'); + strbuf_addstr(buf_path, hex+2); + + scld = safe_create_leading_directories(the_repository, buf_path->buf); + if (scld != SCLD_OK && scld != SCLD_EXISTS) + return -1; + + return 0; +} + +static void my_run_index_pack(struct gh__request_params *params UNUSED, + struct gh__response_status *status, + const struct strbuf *temp_path_pack, + const struct strbuf *temp_path_idx, + struct strbuf *packfile_checksum) +{ + struct child_process ip = CHILD_PROCESS_INIT; + struct strbuf ip_stdout = STRBUF_INIT; + + strvec_push(&ip.args, "git"); + strvec_push(&ip.args, "index-pack"); + + ip.err = -1; + ip.no_stderr = 1; + + /* Skip generating the rev index, we don't need it. */ + strvec_push(&ip.args, "--no-rev-index"); + + strvec_pushl(&ip.args, "-o", temp_path_idx->buf, NULL); + strvec_push(&ip.args, temp_path_pack->buf); + ip.no_stdin = 1; + ip.out = -1; + + if (pipe_command(&ip, NULL, 0, &ip_stdout, 0, NULL, 0)) { + unlink(temp_path_pack->buf); + unlink(temp_path_idx->buf); + strbuf_addf(&status->error_message, + "index-pack failed on '%s'", + temp_path_pack->buf); + /* + * Lets assume that index-pack failed because the + * downloaded file is corrupt (truncated). + * + * Retry it as if the network had dropped. + */ + status->retry = GH__RETRY_MODE__TRANSIENT; + status->ec = GH__ERROR_CODE__INDEX_PACK_FAILED; + goto cleanup; + } + + if (packfile_checksum) { + /* + * stdout from index-pack should have the packfile hash. + * Extract it and use it in the final packfile name. + * + * TODO What kind of validation should we do on the + * TODO string and is there ever any other output besides + * TODO just the checksum ? + */ + strbuf_trim_trailing_newline(&ip_stdout); + + strbuf_addbuf(packfile_checksum, &ip_stdout); + } + +cleanup: + strbuf_release(&ip_stdout); + child_process_clear(&ip); +} + +static void my_finalize_packfile(struct gh__request_params *params, + struct gh__response_status *status, + int b_keep, + const struct strbuf *temp_path_pack, + const struct strbuf *temp_path_idx, + struct strbuf *final_path_pack, + struct strbuf *final_path_idx, + struct strbuf *final_filename) +{ + /* + * Install the .pack and .idx into the ODB pack directory. + * + * We might be racing with other instances of gvfs-helper if + * we, in parallel, both downloaded the exact same packfile + * (with the same checksum SHA) and try to install it at the + * same time. This might happen on Windows where the loser + * can get an EBUSY or EPERM trying to move/rename the + * tempfile into the pack dir, for example. + * + * So, we always install the .pack before the .idx for + * consistency. And only if *WE* created the .pack and .idx + * files, do we create the matching .keep (when requested). + * + * If we get an error and the target files already exist, we + * silently eat the error. Note that finalize_object_file_flags() + * has already munged errno (and it has various creation + * strategies), so we don't bother looking at it. + * + * We use FOF_SKIP_COLLISION_CHECK in case the same packfile was + * attempted for install earlier but got corrupted or failed to + * flush due to a disk issue. This prevents a narrow failure case + * but is better than failing for silly reasons. + */ + if (finalize_object_file_flags(the_repository, + temp_path_pack->buf, final_path_pack->buf, + FOF_SKIP_COLLISION_CHECK) || + finalize_object_file_flags(the_repository, + temp_path_idx->buf, final_path_idx->buf, + FOF_SKIP_COLLISION_CHECK)) { + unlink(temp_path_pack->buf); + unlink(temp_path_idx->buf); + + if (file_exists(final_path_pack->buf) && + file_exists(final_path_idx->buf)) { + trace2_printf("%s: assuming ok for %s", TR2_CAT, final_path_pack->buf); + goto assume_ok; + } + + strbuf_addf(&status->error_message, + "could not install packfile '%s'", + final_path_pack->buf); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PACKFILE; + return; + } + + if (b_keep) { + struct strbuf keep = STRBUF_INIT; + int fd_keep; + + strbuf_addbuf(&keep, final_path_pack); + strbuf_strip_suffix(&keep, ".pack"); + strbuf_addstr(&keep, ".keep"); + + fd_keep = xopen(keep.buf, O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd_keep >= 0) + close(fd_keep); + + strbuf_release(&keep); + } + +assume_ok: + if (params->result_list) { + struct strbuf result_msg = STRBUF_INIT; + + strbuf_addf(&result_msg, "packfile %s", final_filename->buf); + string_list_append(params->result_list, result_msg.buf); + strbuf_release(&result_msg); + } +} + +/* + * Convert the tempfile into a temporary .pack, index it into a temporary .idx + * file, and then install the pair into ODB. + */ +static void install_packfile(struct gh__request_params *params, + struct gh__response_status *status) +{ + struct strbuf temp_path_pack = STRBUF_INIT; + struct strbuf temp_path_idx = STRBUF_INIT; + struct strbuf packfile_checksum = STRBUF_INIT; + struct strbuf final_path_pack = STRBUF_INIT; + struct strbuf final_path_idx = STRBUF_INIT; + struct strbuf final_filename = STRBUF_INIT; + + gh__response_status__zero(status); + + /* + * After the download is complete, we will need to steal the file + * from the tempfile() class (so that it doesn't magically delete + * it when we close the file handle) and then index it. + */ + strbuf_addf(&temp_path_pack, "%s.pack", + get_tempfile_path(params->tempfile)); + strbuf_addf(&temp_path_idx, "%s.idx", + get_tempfile_path(params->tempfile)); + + if (rename_tempfile(¶ms->tempfile, + temp_path_pack.buf) == -1) { + strbuf_addf(&status->error_message, + "could not rename packfile to '%s'", + temp_path_pack.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PACKFILE; + goto cleanup; + } + + my_run_index_pack(params, status, &temp_path_pack, &temp_path_idx, + &packfile_checksum); + if (status->ec != GH__ERROR_CODE__OK) + goto cleanup; + + create_final_packfile_pathnames("vfs", packfile_checksum.buf, NULL, + &final_path_pack, &final_path_idx, + &final_filename); + my_finalize_packfile(params, status, 0, + &temp_path_pack, &temp_path_idx, + &final_path_pack, &final_path_idx, + &final_filename); + +cleanup: + strbuf_release(&temp_path_pack); + strbuf_release(&temp_path_idx); + strbuf_release(&packfile_checksum); + strbuf_release(&final_path_pack); + strbuf_release(&final_path_idx); + strbuf_release(&final_filename); +} + +/* + * bswap.h only defines big endian functions. + * The GVFS Protocol defines fields in little endian. + */ +static inline uint64_t my_get_le64(uint64_t le_val) +{ +#if GIT_BYTE_ORDER == GIT_LITTLE_ENDIAN + return le_val; +#else + return default_bswap64(le_val); +#endif +} + +#define MY_MIN(x,y) (((x) < (y)) ? (x) : (y)) +#define MY_MAX(x,y) (((x) > (y)) ? (x) : (y)) + +/* + * Copy the `nr_bytes_total` from `fd_in` to `fd_out`. + * + * This could be used to extract a single packfile from + * a multipart file, for example. + */ +static int my_copy_fd_len(int fd_in, int fd_out, ssize_t nr_bytes_total) +{ + char buffer[8192]; + + while (nr_bytes_total > 0) { + ssize_t len_to_read = MY_MIN(nr_bytes_total, (ssize_t)sizeof(buffer)); + ssize_t nr_read = xread(fd_in, buffer, len_to_read); + + if (!nr_read) + break; + if (nr_read < 0) + return -1; + + if (write_in_full(fd_out, buffer, nr_read) < 0) + return -1; + + nr_bytes_total -= nr_read; + } + + return 0; +} + +/* + * Copy the `nr_bytes_total` from `fd_in` to `fd_out` AND save the + * final `tail_len` bytes in the given buffer. + * + * This could be used to extract a single packfile from + * a multipart file and read the final SHA into the buffer. + */ +static int my_copy_fd_len_tail(int fd_in, int fd_out, ssize_t nr_bytes_total, + unsigned char *buf_tail, ssize_t tail_len) +{ + memset(buf_tail, 0, tail_len); + + if (my_copy_fd_len(fd_in, fd_out, nr_bytes_total) < 0) + return -1; + + if (nr_bytes_total < tail_len) + return 0; + + /* Reset the position to read the tail */ + lseek(fd_in, -tail_len, SEEK_CUR); + + if (xread(fd_in, (char *)buf_tail, tail_len) != tail_len) + return -1; + + return 0; +} + +/* + * See the protocol document for the per-packfile header. + */ +struct ph { + uint64_t timestamp; + uint64_t pack_len; + uint64_t idx_len; +}; + +/* + * Per-packfile metadata collected during the extraction phase + * of prefetch installation. After all packfiles are extracted + * from the multipack, each entry is handed to index-pack. + */ +struct prefetch_entry { + struct strbuf temp_path_pack; + struct strbuf temp_path_idx; + char hex_checksum[GIT_MAX_HEXSZ + 1]; + timestamp_t timestamp; +}; + +#define PREFETCH_ENTRY_INIT { \ + .temp_path_pack = STRBUF_INIT, \ + .temp_path_idx = STRBUF_INIT, \ + .hex_checksum = {0}, \ + .timestamp = 0, \ +} + +static void prefetch_entry_release(struct prefetch_entry *pe) +{ + strbuf_release(&pe->temp_path_pack); + strbuf_release(&pe->temp_path_idx); +} + +/* + * Extract the next packfile from the multipack into a temp file. + * Populate `entry` with the temp path and checksum, then advance + * the fd past any trailing .idx data. + * + * This is the I/O-bound phase that must run sequentially because + * the multipack is a single stream. + */ +static void extract_packfile_from_multipack( + struct gh__response_status *status, + int fd_multipack, + unsigned short k, + struct prefetch_entry *entry) +{ + struct ph ph; + struct tempfile *tempfile_pack = NULL; + int result = -1; + int b_no_idx_in_multipack; + struct object_id packfile_checksum; + + if (xread(fd_multipack, &ph, sizeof(ph)) != sizeof(ph)) { + strbuf_addf(&status->error_message, + "could not read header for packfile[%d] in multipack", + k); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; + return; + } + + ph.timestamp = my_get_le64(ph.timestamp); + ph.pack_len = my_get_le64(ph.pack_len); + ph.idx_len = my_get_le64(ph.idx_len); + + if (!ph.pack_len) { + strbuf_addf(&status->error_message, + "packfile[%d]: zero length packfile?", k); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; + return; + } + + b_no_idx_in_multipack = (ph.idx_len == maximum_unsigned_value_of_type(uint64_t) || + ph.idx_len == 0); + + /* + * We are going to harden `gvfs-helper` here and ignore the .idx file + * if it is provided and always compute it locally so that we get the + * added verification that `git index-pack` provides. + */ + my_create_tempfile(status, the_repository, 0, "pack", &tempfile_pack, NULL, NULL); + if (!tempfile_pack) + return; + + /* + * Copy the current packfile from the open stream and capture + * the checksum. + * + * TODO This assumes that the checksum is SHA1. Fix this if/when + * TODO Git converts to SHA256. + */ + result = my_copy_fd_len_tail(fd_multipack, + get_tempfile_fd(tempfile_pack), + ph.pack_len, + packfile_checksum.hash, + GIT_SHA1_RAWSZ); + packfile_checksum.algo = GIT_HASH_SHA1; + + if (result < 0) { + strbuf_addf(&status->error_message, + "could not extract packfile[%d] from multipack", + k); + delete_tempfile(&tempfile_pack); + return; + } + strbuf_addstr(&entry->temp_path_pack, get_tempfile_path(tempfile_pack)); + close_tempfile_gently(tempfile_pack); + + oid_to_hex_r(entry->hex_checksum, &packfile_checksum); + entry->timestamp = (timestamp_t)ph.timestamp; + + /* Derive the .idx temp path from the .pack temp path. */ + strbuf_addbuf(&entry->temp_path_idx, &entry->temp_path_pack); + strbuf_strip_suffix(&entry->temp_path_idx, ".pack"); + strbuf_addstr(&entry->temp_path_idx, ".idx"); + + if (!b_no_idx_in_multipack) { + /* + * Server sent the .idx immediately after the .pack in the + * data stream. Skip over it. + */ + if (lseek(fd_multipack, ph.idx_len, SEEK_CUR) < 0) { + strbuf_addf(&status->error_message, + "could not skip index[%d] in multipack", + k); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; + return; + } + } +} + +/* + * Finalize a prefetch packfile after index-pack has already run: + * compute final pathnames and move .pack/.idx/.keep into the ODB. + */ +static void finalize_prefetch_packfile(struct gh__request_params *params, + struct gh__response_status *status, + struct prefetch_entry *entry) +{ + struct strbuf buf_timestamp = STRBUF_INIT; + struct strbuf final_path_pack = STRBUF_INIT; + struct strbuf final_path_idx = STRBUF_INIT; + struct strbuf final_filename = STRBUF_INIT; + + strbuf_addf(&buf_timestamp, "%u", (unsigned int)entry->timestamp); + create_final_packfile_pathnames("prefetch", buf_timestamp.buf, + entry->hex_checksum, + &final_path_pack, &final_path_idx, + &final_filename); + + my_finalize_packfile(params, status, 1, + &entry->temp_path_pack, &entry->temp_path_idx, + &final_path_pack, &final_path_idx, + &final_filename); + + strbuf_release(&buf_timestamp); + strbuf_release(&final_path_pack); + strbuf_release(&final_path_idx); + strbuf_release(&final_filename); +} + +/* + * Context for parallel index-pack execution. + * + * The run_processes_parallel() callbacks are always called from + * the main thread, so no locking is needed for these fields. + */ +struct prefetch_parallel_ctx { + struct prefetch_entry *entries; + unsigned short np; + unsigned short next; + + struct gh__request_params *params; + struct gh__response_status *status; + + struct progress *progress; + int nr_finished; + int nr_installed; +}; + +static int prefetch_get_next_task(struct child_process *cp, + struct strbuf *out UNUSED, + void *pp_cb, + void **pp_task_cb) +{ + struct prefetch_parallel_ctx *ctx = pp_cb; + struct prefetch_entry *entry; + + if (ctx->next >= ctx->np) + return 0; + + entry = &ctx->entries[ctx->next]; + *pp_task_cb = entry; + ctx->next++; + + cp->git_cmd = 1; + strvec_push(&cp->args, "index-pack"); + strvec_push(&cp->args, "--no-rev-index"); + strvec_pushl(&cp->args, "-o", entry->temp_path_idx.buf, NULL); + strvec_push(&cp->args, entry->temp_path_pack.buf); + cp->no_stdin = 1; + cp->no_stdout = 1; + + return 1; +} + +static int prefetch_task_finished(int result, + struct strbuf *out UNUSED, + void *pp_cb, + void *pp_task_cb) +{ + struct prefetch_parallel_ctx *ctx = pp_cb; + struct prefetch_entry *entry = pp_task_cb; + + ctx->nr_finished++; + display_progress(ctx->progress, ctx->nr_finished); + + if (result) { + unlink(entry->temp_path_pack.buf); + unlink(entry->temp_path_idx.buf); + + if (ctx->status->ec == GH__ERROR_CODE__OK) { + strbuf_addf(&ctx->status->error_message, + "index-pack failed on '%s'", + entry->temp_path_pack.buf); + ctx->status->ec = GH__ERROR_CODE__INDEX_PACK_FAILED; + } + return 0; + } + + finalize_prefetch_packfile(ctx->params, ctx->status, entry); + if (ctx->status->ec == GH__ERROR_CODE__OK) + ctx->nr_installed++; + + return 0; +} + +struct keep_files_data { + timestamp_t max_timestamp; + int pos_of_max; + struct string_list *keep_files; +}; + +static void cb_keep_files(const char *full_path, size_t full_path_len UNUSED, + const char *file_path, void *void_data) +{ + struct keep_files_data *data = void_data; + const char *val; + timestamp_t t; + + /* + * We expect prefetch packfiles named like: + * + * prefetch--.keep + */ + if (!skip_prefix(file_path, "prefetch-", &val)) + return; + if (!ends_with(val, ".keep")) + return; + + t = strtol(val, NULL, 10); + if (t > data->max_timestamp) { + data->pos_of_max = data->keep_files->nr; + data->max_timestamp = t; + } + + string_list_append(data->keep_files, full_path); +} + +static void delete_stale_keep_files( + struct gh__request_params *params UNUSED, + struct gh__response_status *status UNUSED) +{ + struct string_list keep_files = STRING_LIST_INIT_DUP; + struct keep_files_data data = { 0, 0, &keep_files }; + size_t k; + + for_each_file_in_pack_dir(gh__global.buf_odb_path.buf, + cb_keep_files, &data); + for (k = 0; k < keep_files.nr; k++) { + if ((ssize_t)k != data.pos_of_max) + unlink(keep_files.items[k].string); + } + + string_list_clear(&keep_files, 0); +} + +/* + * Cut apart the received multipart response into individual packfiles + * and install each one. + */ +static void install_prefetch(struct gh__request_params *params, + struct gh__response_status *status) +{ + static unsigned char v1_h[6] = { 'G', 'P', 'R', 'E', ' ', 0x01 }; + + struct mh { + unsigned char h[6]; + unsigned char np[2]; + }; + + struct mh mh; + unsigned short np; + unsigned short k; + int fd = -1; + int nr_installed = 0; + + struct prefetch_entry *entries = NULL; + + struct strbuf temp_path_mp = STRBUF_INIT; + + /* + * Steal the multi-part file from the tempfile class. + */ + strbuf_addf(&temp_path_mp, "%s.mp", get_tempfile_path(params->tempfile)); + if (rename_tempfile(¶ms->tempfile, temp_path_mp.buf) == -1) { + strbuf_addf(&status->error_message, + "could not rename prefetch tempfile to '%s'", + temp_path_mp.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; + goto cleanup; + } + + fd = git_open_cloexec(temp_path_mp.buf, O_RDONLY); + if (fd == -1) { + strbuf_addf(&status->error_message, + "could not reopen prefetch tempfile '%s'", + temp_path_mp.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; + goto cleanup; + } + + if ((xread(fd, &mh, sizeof(mh)) != sizeof(mh)) || + (memcmp(mh.h, &v1_h, sizeof(mh.h)))) { + strbuf_addstr(&status->error_message, + "invalid prefetch multipart header"); + goto cleanup; + } + + np = (unsigned short)mh.np[0] + ((unsigned short)mh.np[1] << 8); + if (np) + trace2_data_intmax(TR2_CAT, NULL, + "prefetch/packfile_count", np); + + if (!np) + goto cleanup; + + CALLOC_ARRAY(entries, np); + for (k = 0; k < np; k++) { + struct prefetch_entry pe = PREFETCH_ENTRY_INIT; + entries[k] = pe; + } + + /* + * Phase 1: extract all packfiles from the multipack into + * individual temp files. This must be sequential because + * the multipack is a single byte stream. + */ + if (gh__cmd_opts.show_progress) + params->progress = start_progress( + the_repository, "Extracting prefetch packfiles", np); + + for (k = 0; k < np; k++) { + extract_packfile_from_multipack(status, fd, k, &entries[k]); + display_progress(params->progress, k + 1); + if (status->ec != GH__ERROR_CODE__OK) + break; + } + stop_progress(¶ms->progress); + + /* The multipack fd is no longer needed after extraction. */ + close(fd); + fd = -1; + + if (status->ec != GH__ERROR_CODE__OK) + goto cleanup; + + /* + * Phase 2: run index-pack on the extracted packfiles and + * finalize each into the ODB. + * + * When gvfs.prefetchThreads is 1 (the default), process + * packfiles sequentially without any thread infrastructure. + * When set to a higher value, use up to that many concurrent + * index-pack processes. + * + * The entries are already in timestamp order (oldest first), + * so the largest pack—the one that takes the longest—starts + * immediately while the remaining workers cycle through the + * smaller daily/hourly packs. + */ + if (gh__global.prefetch_threads <= 1) { + trace2_data_intmax(TR2_CAT, NULL, + "prefetch/install_mode", 1); + + if (gh__cmd_opts.show_progress) + params->progress = start_progress( + the_repository, + "Installing prefetch packfiles", np); + + for (k = 0; k < np; k++) { + my_run_index_pack(params, status, + &entries[k].temp_path_pack, + &entries[k].temp_path_idx, + NULL); + if (status->ec == GH__ERROR_CODE__OK) { + finalize_prefetch_packfile(params, status, + &entries[k]); + if (status->ec == GH__ERROR_CODE__OK) + nr_installed++; + } + display_progress(params->progress, k + 1); + if (status->ec != GH__ERROR_CODE__OK) + break; + } + stop_progress(¶ms->progress); + } else { + struct prefetch_parallel_ctx pctx = { + .entries = entries, + .np = np, + .next = 0, + .params = params, + .status = status, + .nr_finished = 0, + .nr_installed = 0, + }; + struct run_process_parallel_opts pp_opts = { + .tr2_category = TR2_CAT, + .tr2_label = "prefetch/index-pack", + .processes = MY_MIN(np, gh__global.prefetch_threads), + .get_next_task = prefetch_get_next_task, + .task_finished = prefetch_task_finished, + .data = &pctx, + }; + + trace2_data_intmax(TR2_CAT, NULL, + "prefetch/install_mode", + gh__global.prefetch_threads); + + if (gh__cmd_opts.show_progress) + pctx.progress = start_progress( + the_repository, + "Installing prefetch packfiles", np); + + run_processes_parallel(&pp_opts); + + stop_progress(&pctx.progress); + nr_installed = pctx.nr_installed; + } + + if (nr_installed) + delete_stale_keep_files(params, status); + +cleanup: + if (entries) { + for (k = 0; k < np; k++) + prefetch_entry_release(&entries[k]); + free(entries); + } + + if (fd != -1) + close(fd); + + unlink(temp_path_mp.buf); + strbuf_release(&temp_path_mp); +} + +/* + * Wrapper for read_loose_object() to read and verify the hash of a + * loose object, and discard the contents buffer. + * + * Returns 0 on success, negative on error (details may be written to stderr). + */ +static int verify_loose_object(const char *path, + const struct object_id *expected_oid) +{ + enum object_type type; + void *contents = NULL; + size_t size; + int ret; + struct object_info oi = OBJECT_INFO_INIT; + struct object_id real_oid = *null_oid(the_hash_algo); + oi.typep = &type; + oi.sizep = &size; + + ret = read_loose_object(the_repository, path, expected_oid, &real_oid, &contents, &oi); + free(contents); + + return ret; +} + +/* + * Convert the tempfile into a permanent loose object in the ODB. + */ +static void install_loose(struct gh__request_params *params, + struct gh__response_status *status) +{ + struct strbuf tmp_path = STRBUF_INIT; + struct strbuf loose_path = STRBUF_INIT; + + gh__response_status__zero(status); + + /* + * close tempfile to steal ownership away from tempfile class. + */ + strbuf_addstr(&tmp_path, get_tempfile_path(params->tempfile)); + close_tempfile_gently(params->tempfile); + + /* + * Compute the hash of the received content (while it is still + * in a temp file) and verify that it matches the OID that we + * requested and was not corrupted. + */ + if (verify_loose_object(tmp_path.buf, ¶ms->loose_oid)) { + strbuf_addf(&status->error_message, + "hash failed for received loose object '%s'", + oid_to_hex(¶ms->loose_oid)); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_LOOSE; + goto cleanup; + } + + /* + * Try to install the tempfile as the actual loose object. + * + * If the loose object already exists, finalize_object_file() + * will NOT overwrite/replace it. It will silently eat the + * EEXIST error and unlink the tempfile as it if was + * successful. We just let it lie to us. + * + * Since our job is to back-fill missing objects needed by a + * foreground git process -- git should have called + * oid_object_info_extended() and loose_object_info() BEFORE + * asking us to download the missing object. So if we get a + * collision we have to assume something else is happening in + * parallel and we lost the race. And that's OK. + */ + if (create_loose_pathname_in_odb(&loose_path, ¶ms->loose_oid)) { + strbuf_addf(&status->error_message, + "cannot create directory for loose object '%s'", + loose_path.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_LOOSE; + goto cleanup; + } + + /* + * We skip collision check because the loose object in the target + * may be corrupt and we should override it with a better value + * instead of failing at this point. + * + * See https://github.com/microsoft/git/issues/837 + */ + if (finalize_object_file_flags(the_repository, + tmp_path.buf, loose_path.buf, + FOF_SKIP_COLLISION_CHECK)) { + unlink(tmp_path.buf); + strbuf_addf(&status->error_message, + "could not install loose object '%s'", + loose_path.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_LOOSE; + goto cleanup; + } + + if (params->result_list) { + struct strbuf result_msg = STRBUF_INIT; + + strbuf_addf(&result_msg, "loose %s", + oid_to_hex(¶ms->loose_oid)); + string_list_append(params->result_list, result_msg.buf); + strbuf_release(&result_msg); + } + +cleanup: + strbuf_release(&tmp_path); + strbuf_release(&loose_path); +} + +static void install_result(struct gh__request_params *params, + struct gh__response_status *status) +{ + if (params->objects_mode == GH__OBJECTS_MODE__PREFETCH) { + /* + * The "gvfs/prefetch" API is the only thing that sends + * these multi-part packfiles. According to the protocol + * documentation, they will have this x- content type. + * + * However, it appears that there is a BUG in the origin + * server causing it to sometimes send "text/html" instead. + * So, we silently handle both. + */ + if (!strcmp(status->content_type.buf, + "application/x-gvfs-timestamped-packfiles-indexes")) { + install_prefetch(params, status); + return; + } + + if (!strcmp(status->content_type.buf, "text/html")) { + install_prefetch(params, status); + return; + } + } else { + if (!strcmp(status->content_type.buf, "application/x-git-packfile")) { + assert(params->b_is_post); + assert(params->objects_mode == GH__OBJECTS_MODE__POST); + + install_packfile(params, status); + return; + } + + if (!strcmp(status->content_type.buf, + "application/x-git-loose-object")) { + /* + * We get these for "gvfs/objects" GET and POST requests. + * + * Note that this content type is singular, not plural. + */ + install_loose(params, status); + return; + } + } + + strbuf_addf(&status->error_message, + "install_result: received unknown content-type '%s'", + status->content_type.buf); + status->ec = GH__ERROR_CODE__UNEXPECTED_CONTENT_TYPE; +} + +/* + * Our wrapper to initialize the HTTP layer. + * + * We always use the real origin server, not the cache-server, when + * initializing the http/curl layer. + */ +static void gh_http_init(void) +{ + if (gh__global.http_is_initialized) + return; + + http_init(gh__global.remote, gh__global.main_url, 0); + gh__global.http_is_initialized = 1; +} + +static void gh_http_cleanup(void) +{ + if (!gh__global.http_is_initialized) + return; + + http_cleanup(); + gh__global.http_is_initialized = 0; +} + +/* + * buffer has ": [\r]\n" + */ +static void parse_resp_hdr_1(const char *buffer, size_t size, size_t nitems, + struct strbuf *key, struct strbuf *value) +{ + const char *end = buffer + (size * nitems); + const char *p; + + p = strchr(buffer, ':'); + + strbuf_setlen(key, 0); + strbuf_add(key, buffer, (p - buffer)); + + p++; /* skip ':' */ + p++; /* skip ' ' */ + + strbuf_setlen(value, 0); + strbuf_add(value, p, (end - p)); + strbuf_trim_trailing_newline(value); +} + +static void parse_gvfs_response_header( + const char *buffer, size_t size, size_t nitems, + struct gh__azure_throttle *azure, struct strbuf *e2eid, + enum gh__server_type server_type) +{ + if (starts_with(buffer, "X-RateLimit-")) { + struct strbuf key = STRBUF_INIT; + struct strbuf val = STRBUF_INIT; + + parse_resp_hdr_1(buffer, size, nitems, &key, &val); + + /* + * The following X- headers are specific to AzureDevOps. + * Other servers have similar sets of values, but I haven't + * compared them in depth. + */ + // trace2_printf("%s: Throttle: %s %s", TR2_CAT, key.buf, val.buf); + + if (!strcmp(key.buf, "X-RateLimit-Resource")) { + /* + * The name of the resource that is complaining. + * Just log it because we can't do anything with it. + */ + strbuf_setlen(&key, 0); + strbuf_addstr(&key, "ratelimit/resource"); + strbuf_addstr(&key, + gh__server_type_label[server_type]); + + trace2_data_string(TR2_CAT, NULL, key.buf, val.buf); + } + + else if (!strcmp(key.buf, "X-RateLimit-Delay")) { + /* + * The amount of delay added to our response. + * Just log it because we can't do anything with it. + */ + unsigned long tarpit_delay_ms; + + strbuf_setlen(&key, 0); + strbuf_addstr(&key, "ratelimit/delay_ms"); + strbuf_addstr(&key, + gh__server_type_label[server_type]); + + git_parse_ulong(val.buf, &tarpit_delay_ms); + + trace2_data_intmax(TR2_CAT, NULL, key.buf, tarpit_delay_ms); + } + + else if (!strcmp(key.buf, "X-RateLimit-Limit")) { + /* + * The resource limit/quota before we get a 429. + */ + git_parse_ulong(val.buf, &azure->tstu_limit); + } + + else if (!strcmp(key.buf, "X-RateLimit-Remaining")) { + /* + * The amount of our quota remaining. When zero, we + * should get 429s on futher requests until the reset + * time. + */ + git_parse_ulong(val.buf, &azure->tstu_remaining); + } + + else if (!strcmp(key.buf, "X-RateLimit-Reset")) { + /* + * The server gave us a time-in-seconds-since-the-epoch + * for when our quota will be reset (if we stop all + * activity right now). + * + * Checkpoint the local system clock so we can do some + * sanity checks on any clock skew. Also, since we get + * the headers before we get the content, we can adjust + * our delay to compensate for the full download time. + */ + unsigned long now = time(NULL); + unsigned long reset_time; + + git_parse_ulong(val.buf, &reset_time); + if (reset_time > now) + azure->reset_sec = reset_time - now; + } + + strbuf_release(&key); + strbuf_release(&val); + } + + else if (starts_with(buffer, "Retry-After")) { + struct strbuf key = STRBUF_INIT; + struct strbuf val = STRBUF_INIT; + + parse_resp_hdr_1(buffer, size, nitems, &key, &val); + + /* + * We get this header with a 429 and 503 and possibly a 30x. + * + * Curl does have CURLINFO_RETRY_AFTER that nicely parses and + * normalizes the value (and supports HTTP/1.1 usage), but it + * is not present yet in the version shipped with the Mac, so + * we do it directly here. + */ + git_parse_ulong(val.buf, &azure->retry_after_sec); + + strbuf_release(&key); + strbuf_release(&val); + } + + else if (starts_with(buffer, "X-VSS-E2EID")) { + struct strbuf key = STRBUF_INIT; + + /* + * Capture the E2EID as it goes by, but don't log it until we + * know the request result. + */ + parse_resp_hdr_1(buffer, size, nitems, &key, e2eid); + + strbuf_release(&key); + } +} + +static size_t parse_resp_hdr(char *buffer, size_t size, size_t nitems, + void *void_params) +{ + struct gh__request_params *params = void_params; + struct gh__azure_throttle *azure = + &gh__global_throttle[params->server_type]; + + parse_gvfs_response_header(buffer, size, nitems, azure, + ¶ms->e2eid, params->server_type); + + return nitems * size; +} + +/* + * Wait "duration" seconds and drive the progress mechanism. + * + * We spin slightly faster than we need to to keep the progress bar + * drawn (especially if the user presses return while waiting) and to + * compensate for delay factors built into the progress class (which + * might wait for 2 seconds before drawing the first message). + */ +static void do_throttle_spin(struct gh__request_params *params, + const char *tr2_label, + const char *progress_msg, + int duration) +{ + struct strbuf region = STRBUF_INIT; + struct progress *progress = NULL; + unsigned long begin = time(NULL); + unsigned long now = begin; + unsigned long end = begin + duration; + + strbuf_addstr(®ion, tr2_label); + strbuf_addstr(®ion, gh__server_type_label[params->server_type]); + trace2_region_enter(TR2_CAT, region.buf, NULL); + + if (gh__cmd_opts.show_progress) + progress = start_progress(the_repository, progress_msg, duration); + + while (now < end) { + display_progress(progress, (now - begin)); + + sleep_millisec(100); + + now = time(NULL); + } + + display_progress(progress, duration); + stop_progress(&progress); + + trace2_region_leave(TR2_CAT, region.buf, NULL); + strbuf_release(®ion); +} + +/* + * Delay the outbound request if necessary in response to previous throttle + * blockages or hints. Throttle data is somewhat orthogonal to the status + * results from any previous request and/or the request params of the next + * request. + * + * Note that the throttle info also is cross-process information, such as + * 2 concurrent fetches in 2 different terminal windows to the same server + * will be sharing the same server quota. These could be coordinated too, + * so that a blockage received in one process would prevent the other + * process from starting another request (and also blocked or extending + * the delay interval). We're NOT going to do that level of integration. + * We will let both processes independently attempt the next request. + * This may cause us to miss the end-of-quota boundary if the server + * extends it because of the second request. + * + * TODO Should we have a max-wait option and then return a hard-error + * TODO of some type? + */ +static void do_throttle_wait(struct gh__request_params *params, + struct gh__response_status *status UNUSED) +{ + struct gh__azure_throttle *azure = + &gh__global_throttle[params->server_type]; + + if (azure->retry_after_sec) { + /* + * We were given a hard delay (such as after a 429). + * Spin until the requested time. + */ + do_throttle_spin(params, "throttle/hard", + "Waiting on hard throttle (sec)", + azure->retry_after_sec); + return; + } + + if (azure->reset_sec > 0) { + /* + * We were given a hint that we are overloading + * the server. Voluntarily backoff (before we + * get tarpitted or blocked). + */ + do_throttle_spin(params, "throttle/soft", + "Waiting on soft throttle (sec)", + azure->reset_sec); + return; + } + + if (params->k_transient_delay_sec) { + /* + * Insert an arbitrary delay before retrying after a + * transient (network) failure. + */ + do_throttle_spin(params, "throttle/transient", + "Waiting to retry after network error (sec)", + params->k_transient_delay_sec); + return; + } +} + +static void set_main_creds_on_slot(struct active_request_slot *slot, + const struct credential *creds) +{ + assert(creds == &gh__global.main_creds); + + /* + * When talking to the main/origin server, we have 3 modes + * of operation: + * + * [1] The initial request is sent without loading creds + * and with ANY-AUTH set. (And the `":"` is a magic + * value.) + * + * This allows libcurl to negotiate for us if it can. + * For example, this allows NTLM to work by magic and + * we get 200s without ever seeing a 401. If libcurl + * cannot negotiate for us, it gives us a 401 (and all + * of the 401 code in this file responds to that). + * + * [2] A 401 retry will load the main creds and try again. + * This causes `creds->username`to be non-NULL (even + * if refers to a zero-length string). And we assume + * BASIC Authentication. (And a zero-length username + * is a convention for PATs, but then sometimes users + * put the PAT in their `username` field and leave the + * `password` field blank. And that works too.) + * + * [3] Subsequent requests on the same connection use + * whatever worked before. + */ + if (creds && creds->username) { + curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_easy_setopt(slot->curl, CURLOPT_USERNAME, creds->username); + curl_easy_setopt(slot->curl, CURLOPT_PASSWORD, creds->password); + } else { + curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY); + curl_easy_setopt(slot->curl, CURLOPT_USERPWD, ":"); + } +} + +static void set_cache_server_creds_on_slot(struct active_request_slot *slot, + const struct credential *creds) +{ + assert(creds == &gh__global.cache_creds); + assert(creds->username); + + /* + * Things are weird when talking to a cache-server: + * + * [1] They don't send 401s on an auth error, rather they send + * a 400 (with a nice human-readable string in the html body). + * This prevents libcurl from doing any negotiation for us. + * + * [2] Cache-servers don't manage their own passwords, but + * rather require us to send the Basic Authentication + * username & password that we would send to the main + * server. (So yes, we have to get creds validated + * against the main server creds and substitute them when + * talking to the cache-server.) + * + * This means that: + * + * [a] We cannot support cache-servers that want to use NTLM. + * + * [b] If we want to talk to a cache-server, we have get the + * Basic Auth creds for the main server. And this may be + * problematic if the libcurl and/or the credential manager + * insists on using NTLM and prevents us from getting them. + * + * So we never try AUTH-ANY and force Basic Auth (if possible). + */ + if (creds && creds->username) { + curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_easy_setopt(slot->curl, CURLOPT_USERNAME, creds->username); + curl_easy_setopt(slot->curl, CURLOPT_PASSWORD, creds->password); + } +} + +/* + * Do a single HTTP request WITHOUT robust-retry, auth-retry or fallback. + */ +static void do_req(const char *url_base, + const char *url_component, + const struct credential *creds, + struct gh__request_params *params, + struct gh__response_status *status) +{ + struct active_request_slot *slot; + struct slot_results results; + struct strbuf rest_url = STRBUF_INIT; + + gh__response_status__zero(status); + + if (params->b_write_to_file) { + /* Delete dirty tempfile from a previous attempt. */ + if (params->tempfile) + delete_tempfile(¶ms->tempfile); + + my_create_tempfile(status, the_repository, 1, NULL, ¶ms->tempfile, NULL, NULL); + if (!params->tempfile || status->ec != GH__ERROR_CODE__OK) + return; + } else { + /* Guard against caller using dirty buffer */ + strbuf_setlen(params->buffer, 0); + } + + end_url_with_slash(&rest_url, url_base); + strbuf_addstr(&rest_url, url_component); + + do_throttle_wait(params, status); + gh__azure_throttle__zero(&gh__global_throttle[params->server_type]); + + slot = get_active_slot(); + slot->results = &results; + + curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0L); /* not a HEAD request */ + curl_easy_setopt(slot->curl, CURLOPT_URL, rest_url.buf); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, params->headers); + if (curl_version_info(CURLVERSION_NOW)->version_num < 0x074b00) + /* + * cURL 7.75.0 allows headers to be parsed even when + * `CURLOPT_FAILONERROR` is enabled and the HTTP result code + * indicates an error. This is the behavior expected by + * `gvfs-helper`. + * + * On older cURL versions, `gvfs-helper` still needs to parse + * the HTTP headers and therefore needs to _not_ fail upon + * HTTP result codes indicating errors; For newer cURL + * versions, we still prefer to enable `FAILONERROR`. + */ + curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, (long)0); + + if (params->b_is_post) { + curl_easy_setopt(slot->curl, CURLOPT_POST, 1L); + curl_easy_setopt(slot->curl, CURLOPT_ENCODING, NULL); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, + params->post_payload->buf); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, + (long)params->post_payload->len); + } else { + curl_easy_setopt(slot->curl, CURLOPT_POST, 0L); + } + + if (params->b_write_to_file) { + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); + curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, + (void*)params->tempfile->fp); + } else { + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, + fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_FILE, params->buffer); + } + + curl_easy_setopt(slot->curl, CURLOPT_HEADERFUNCTION, parse_resp_hdr); + curl_easy_setopt(slot->curl, CURLOPT_HEADERDATA, params); + + if (params->server_type == GH__SERVER_TYPE__MAIN) + set_main_creds_on_slot(slot, creds); + else + set_cache_server_creds_on_slot(slot, creds); + + if (params->progress_base_phase2_msg.len || + params->progress_base_phase3_msg.len) { + curl_easy_setopt(slot->curl, CURLOPT_XFERINFOFUNCTION, + gh__curl_progress_cb); + curl_easy_setopt(slot->curl, CURLOPT_XFERINFODATA, params); + curl_easy_setopt(slot->curl, CURLOPT_NOPROGRESS, 0L); + } else { + curl_easy_setopt(slot->curl, CURLOPT_NOPROGRESS, 1L); + } + + if (gh__global.connect_timeout_ms) + curl_easy_setopt(slot->curl, CURLOPT_CONNECTTIMEOUT_MS, + gh__global.connect_timeout_ms); + + gh__run_one_slot(slot, params, status); + strbuf_release(&rest_url); +} + +/* + * Compute the delay for the nth attempt. + * + * No delay for the first attempt. Then use a normal exponential backoff + * starting from 8. + */ +static int compute_transient_delay(int attempt) +{ + int v; + + if (attempt < 1) + return 0; + + /* + * Let 8K be our hard limit (for integer overflow protection). + * That's over 2 hours. This is 8<<10. + */ + if (attempt > 10) + attempt = 10; + + v = 8 << (attempt - 1); + + if (v > gh__cmd_opts.max_transient_backoff_sec) + v = gh__cmd_opts.max_transient_backoff_sec; + + return v; +} + +static void gvfs_advice_on_retry(enum gh__error_code ec) +{ + static int advice_given = 0; + + if (advice_given) + return; + advice_given = 1; + + if (ec < GH__ERROR_CODE__HTTP_ERROR_LIMIT) { + advise_if_enabled(ADVICE_GVFS_HELPER_TRANSIENT_RETRY, + "GVFS Protocol network requests are failing. This is\n" + "likely caused by a service outage or unstable network\n" + "connection. Check your local network or contact your\n" + "engineering systems team for assistance."); + return; + } + + if (gvfs_shared_cache_pathname.len) { + advise_if_enabled(ADVICE_GVFS_HELPER_TRANSIENT_RETRY, + "These retries may hint towards issues with your disk or\n" + "shared object cache. Check to see if your disk is full.\n" + "If your disk has space, then your shared object cache\n" + "may have corrupt files. Push all local branches then\n" + "delete '%s'\n" + "and run 'git fetch' to reload the cache.", + gvfs_shared_cache_pathname.buf); + } else { + advise_if_enabled(ADVICE_GVFS_HELPER_TRANSIENT_RETRY, + "These retries may hint towards issues with your disk.\n" + "Check to see if your disk is full. Note also that you\n" + "do not have a gvfs.sharedCache config, which is not\n" + "normal. You may need to delete and reclone this repo."); + } +} + +/* + * Robustly make an HTTP request. Retry if necessary to hide common + * transient network errors and/or 429 blockages. + * + * For a transient (network) failure (where we do not have a throttle + * delay factor), we should insert a small delay to let the network + * recover. The outage might be because the VPN dropped, or the + * machine went to sleep or something and we want to give the network + * time to come back up. Insert AI here :-) + */ +static void do_req__with_robust_retry(const char *url_base, + const char *url_component, + const struct credential *creds, + struct gh__request_params *params, + struct gh__response_status *status) +{ + for (params->k_attempt = 0; + params->k_attempt < gh__cmd_opts.max_retries + 1; + params->k_attempt++) { + + do_req(url_base, url_component, creds, params, status); + + switch (status->retry) { + default: + case GH__RETRY_MODE__SUCCESS: + case GH__RETRY_MODE__HTTP_401: /* caller does auth-retry */ + case GH__RETRY_MODE__HARD_FAIL: + case GH__RETRY_MODE__FAIL_404: + return; + + case GH__RETRY_MODE__HTTP_429: + case GH__RETRY_MODE__HTTP_503: + /* + * We should have gotten a "Retry-After" header with + * these and that gives us the wait time. If not, + * fallthru and use the backoff delay. + */ + if (gh__global_throttle[params->server_type].retry_after_sec) + continue; + /*fallthru*/ + + case GH__RETRY_MODE__TRANSIENT: + /* + * Give advice for common reasons this could happen: + */ + gvfs_advice_on_retry(status->ec); + params->k_transient_delay_sec = + compute_transient_delay(params->k_attempt); + continue; + } + } +} + +static void do_req__to_main(const char *url_component, + struct gh__request_params *params, + struct gh__response_status *status) +{ + params->server_type = GH__SERVER_TYPE__MAIN; + + /* + * When talking to the main Git server, we DO NOT preload the + * creds before the first request. + */ + + do_req__with_robust_retry(gh__global.main_url, url_component, + &gh__global.main_creds, + params, status); + + if (status->retry == GH__RETRY_MODE__HTTP_401) { + refresh_main_creds(); + + do_req__with_robust_retry(gh__global.main_url, url_component, + &gh__global.main_creds, + params, status); + } + + if (status->retry == GH__RETRY_MODE__SUCCESS) + approve_main_creds(); +} + +static void do_req__to_cache_server(const char *url_component, + struct gh__request_params *params, + struct gh__response_status *status) +{ + params->server_type = GH__SERVER_TYPE__CACHE; + + /* + * When talking to a cache-server, DO force load the creds. + * This implicitly preloads the creds to the main server. + */ + synthesize_cache_server_creds(); + + do_req__with_robust_retry(gh__global.cache_server_url, url_component, + &gh__global.cache_creds, + params, status); + + if (status->retry == GH__RETRY_MODE__HTTP_401) { + refresh_cache_server_creds(); + + do_req__with_robust_retry(gh__global.cache_server_url, + url_component, + &gh__global.cache_creds, + params, status); + } + + if (status->retry == GH__RETRY_MODE__SUCCESS) + approve_cache_server_creds(); +} + +/* + * Try the cache-server (if configured) then fall-back to the main Git server. + */ +static void do_req__with_fallback(const char *url_component, + struct gh__request_params *params, + struct gh__response_status *status) +{ +retry_backup: + if (gh__global.cache_server_url && + params->b_permit_cache_server_if_defined) { + do_req__to_cache_server(url_component, params, status); + + if (status->retry == GH__RETRY_MODE__SUCCESS) + return; + + if (!gh__cmd_opts.try_fallback) + return; + + /* + * If we overrode the cache-server using a custom key, + * then fall back to the regular cache server on failure. + */ + if (gh__global.cache_server_url_backup) { + reset_cache_server(); + goto retry_backup; + } + + /* + * The cache-server shares creds with the main Git server, + * so if our creds failed against the cache-server, they + * will also fail against the main Git server. We just let + * this fail. + * + * Falling-back would likely just cause the 3rd (or maybe + * 4th) cred prompt. + */ + if (status->retry == GH__RETRY_MODE__HTTP_401) + return; + } + + do_req__to_main(url_component, params, status); +} + +/* + * Call "gvfs/config" REST API. + * + * Return server's response buffer. This is probably a raw JSON string. + */ +static void do__http_get__simple_endpoint(struct gh__response_status *status, + struct strbuf *response, + const char *endpoint, + const char *tr2_label) +{ + struct gh__request_params params = GH__REQUEST_PARAMS_INIT; + + strbuf_addstr(¶ms.tr2_label, tr2_label); + + params.b_is_post = 0; + params.b_write_to_file = 0; + /* cache-servers do not handle gvfs/config REST calls */ + params.b_permit_cache_server_if_defined = 0; + params.buffer = response; + params.objects_mode = GH__OBJECTS_MODE__NONE; + + params.object_count = 1; /* a bit of a lie */ + + /* + * "X-TFS-FedAuthRedirect: Suppress" disables the 302 + 203 redirect + * sequence to a login page and forces the main Git server to send a + * normal 401. + */ + params.headers = http_copy_default_headers(); + params.headers = curl_slist_append(params.headers, + "X-TFS-FedAuthRedirect: Suppress"); + params.headers = curl_slist_append(params.headers, + "Pragma: no-cache"); + append_session_id_header(¶ms.headers); + + if (gh__cmd_opts.show_progress) { + /* + * gvfs/config has a very small reqest payload, so I don't + * see any need to report progress on the upload side of + * the GET. So just report progress on the download side. + */ + strbuf_addf(¶ms.progress_base_phase3_msg, + "Receiving %s", endpoint); + } + + do_req__with_fallback(endpoint, ¶ms, status); + + gh__request_params__release(¶ms); +} + +static void do__http_get__gvfs_config(struct gh__response_status *status, + struct strbuf *config_data) +{ + do__http_get__simple_endpoint(status, config_data, "gvfs/config", + "GET/config"); +} + +static void setup_gvfs_objects_progress(struct gh__request_params *params, + unsigned long num, unsigned long den) +{ + if (!gh__cmd_opts.show_progress) + return; + + if (params->b_is_post) { + strbuf_addf(¶ms->progress_base_phase3_msg, + "Receiving packfile %ld/%ld with %ld objects", + num, den, params->object_count); + } + /* If requesting only one object, then do not show progress */ +} + +/* + * Call "gvfs/objects/" REST API to fetch a loose object + * and write it to the ODB. + */ +static void do__http_get__gvfs_object(struct gh__response_status *status, + const struct object_id *oid, + unsigned long l_num, unsigned long l_den, + struct string_list *result_list) +{ + struct gh__request_params params = GH__REQUEST_PARAMS_INIT; + struct strbuf component_url = STRBUF_INIT; + + gh__response_status__zero(status); + + strbuf_addf(&component_url, "gvfs/objects/%s", oid_to_hex(oid)); + + strbuf_addstr(¶ms.tr2_label, "GET/objects"); + + params.b_is_post = 0; + params.b_write_to_file = 1; + params.b_permit_cache_server_if_defined = 1; + params.objects_mode = GH__OBJECTS_MODE__GET; + + params.object_count = 1; + + params.result_list = result_list; + + params.headers = http_copy_default_headers(); + params.headers = curl_slist_append(params.headers, + "X-TFS-FedAuthRedirect: Suppress"); + params.headers = curl_slist_append(params.headers, + "Pragma: no-cache"); + append_session_id_header(¶ms.headers); + + oidcpy(¶ms.loose_oid, oid); + + setup_gvfs_objects_progress(¶ms, l_num, l_den); + + update_cache_server_for_verb(GET); + do_req__with_fallback(component_url.buf, ¶ms, status); + reset_cache_server(); + + gh__request_params__release(¶ms); + strbuf_release(&component_url); +} + +/* + * Call "gvfs/objects" POST REST API to fetch a batch of objects + * from the OIDSET. Normal, this is results in a packfile containing + * `nr_wanted_in_block` objects. And we return the number actually + * consumed (along with the filename of the resulting packfile). + * + * However, if we only have 1 oid (remaining) in the OIDSET, the + * server *MAY* respond to our POST with a loose object rather than + * a packfile with 1 object. + * + * Append a message to the result_list describing the result. + * + * Return the number of OIDs consumed from the OIDSET. + */ +static void do__http_post__gvfs_objects(struct gh__response_status *status, + struct oidset_iter *iter, + unsigned long nr_wanted_in_block, + int j_pack_num, int j_pack_den, + struct string_list *result_list, + unsigned long *nr_oid_taken) +{ + struct json_writer jw_req = JSON_WRITER_INIT; + struct gh__request_params params = GH__REQUEST_PARAMS_INIT; + + gh__response_status__zero(status); + + params.object_count = build_json_payload__gvfs_objects( + &jw_req, iter, nr_wanted_in_block, ¶ms.loose_oid); + *nr_oid_taken = params.object_count; + + strbuf_addstr(¶ms.tr2_label, "POST/objects"); + + params.b_is_post = 1; + params.b_write_to_file = 1; + params.b_permit_cache_server_if_defined = 1; + params.objects_mode = GH__OBJECTS_MODE__POST; + + params.post_payload = &jw_req.json; + + params.result_list = result_list; + + params.headers = http_copy_default_headers(); + params.headers = curl_slist_append(params.headers, + "X-TFS-FedAuthRedirect: Suppress"); + params.headers = curl_slist_append(params.headers, + "Pragma: no-cache"); + params.headers = curl_slist_append(params.headers, + "Content-Type: application/json"); + append_session_id_header(¶ms.headers); + + /* + * If our POST contains more than one object, we want the + * server to send us a packfile. We DO NOT want the non-standard + * concatenated loose object format, so we DO NOT send: + * "Accept: application/x-git-loose-objects" (plural) + * + * However, if the payload only requests 1 OID, the server + * will send us a single loose object instead of a packfile, + * so we ACK that and send: + * "Accept: application/x-git-loose-object" (singular) + */ + params.headers = curl_slist_append(params.headers, + "Accept: application/x-git-packfile"); + params.headers = curl_slist_append(params.headers, + "Accept: application/x-git-loose-object"); + + setup_gvfs_objects_progress(¶ms, j_pack_num, j_pack_den); + + update_cache_server_for_verb(POST); + do_req__with_fallback("gvfs/objects", ¶ms, status); + reset_cache_server(); + + gh__request_params__release(¶ms); + jw_release(&jw_req); +} + +struct find_last_data { + timestamp_t timestamp; + int nr_files; +}; + +static void cb_find_last(const char *full_path UNUSED, size_t full_path_len UNUSED, + const char *file_path, void *void_data) +{ + struct find_last_data *data = void_data; + const char *val; + timestamp_t t; + + if (!skip_prefix(file_path, "prefetch-", &val)) + return; + if (!ends_with(val, ".pack")) + return; + + data->nr_files++; + + /* + * We expect prefetch packfiles named like: + * + * prefetch--.pack + */ + t = strtol(val, NULL, 10); + + data->timestamp = MY_MAX(t, data->timestamp); +} + +/* + * Find the server timestamp on the last prefetch packfile that + * we have in the ODB. + * + * TODO I'm going to assume that all prefetch packs are created + * TODO equal and take the one with the largest t value. + * TODO + * TODO Or should we look for one marked with .keep ? + * + * TODO Alternatively, should we maybe get the 2nd largest? + * TODO (Or maybe subtract an hour delta from the largest?) + * TODO + * TODO Since each cache-server maintains its own set of prefetch + * TODO packs (such that 2 requests may hit 2 different + * TODO load-balanced servers and get different answers (with or + * TODO without clock-skew issues)), is it possible for us to miss + * TODO the absolute fringe of new commits and trees? + * TODO + * TODO That is, since the cache-server generates hourly prefetch + * TODO packs, we could do a prefetch and be up-to-date, but then + * TODO do the main fetch and hit a different cache/main server + * TODO and be behind by as much as an hour and have to demand- + * TODO load the commits/trees. + * + * TODO Alternatively, should we compare the last timestamp found + * TODO with "now" and silently do nothing if within an epsilon? + */ +static void find_last_prefetch_timestamp(timestamp_t *last) +{ + struct find_last_data data; + + memset(&data, 0, sizeof(data)); + + for_each_file_in_pack_dir(gh__global.buf_odb_path.buf, cb_find_last, &data); + + *last = data.timestamp; +} + +/* + * Call "gvfs/prefetch[?lastPackTimestamp=]" REST API to + * fetch a series of packfiles and write them to the ODB. + * + * Return a list of packfile names. + */ +static void do__http_get__gvfs_prefetch(struct gh__response_status *status, + timestamp_t seconds_since_epoch, + struct string_list *result_list) +{ + struct gh__request_params params = GH__REQUEST_PARAMS_INIT; + struct strbuf component_url = STRBUF_INIT; + + gh__response_status__zero(status); + + strbuf_addstr(&component_url, "gvfs/prefetch"); + + if (!seconds_since_epoch) + find_last_prefetch_timestamp(&seconds_since_epoch); + if (seconds_since_epoch) + strbuf_addf(&component_url, "?lastPackTimestamp=%"PRItime, + seconds_since_epoch); + + trace2_data_intmax(TR2_CAT, the_repository, + "prefetch/since", + seconds_since_epoch); + + params.b_is_post = 0; + params.b_write_to_file = 1; + params.b_permit_cache_server_if_defined = 1; + params.objects_mode = GH__OBJECTS_MODE__PREFETCH; + + params.object_count = -1; + + params.result_list = result_list; + + params.headers = http_copy_default_headers(); + params.headers = curl_slist_append(params.headers, + "X-TFS-FedAuthRedirect: Suppress"); + params.headers = curl_slist_append(params.headers, + "Pragma: no-cache"); + params.headers = curl_slist_append(params.headers, + "Accept: application/x-gvfs-timestamped-packfiles-indexes"); + append_session_id_header(¶ms.headers); + + if (gh__cmd_opts.show_progress) + strbuf_addf(¶ms.progress_base_phase3_msg, + "Prefetch %"PRItime" (%s)", + seconds_since_epoch, + show_date(seconds_since_epoch, 0, + DATE_MODE(ISO8601))); + + update_cache_server_for_verb(PREFETCH); + do_req__with_fallback(component_url.buf, ¶ms, status); + reset_cache_server(); + + gh__request_params__release(¶ms); + strbuf_release(&component_url); +} + +/* + * Drive one or more HTTP GET requests to fetch the objects + * in the given OIDSET. These are received into loose objects. + * + * Accumulate results for each request in `result_list` until we get a + * hard error and have to stop. + */ +static void do__http_get__fetch_oidset(struct gh__response_status *status, + struct oidset *oids, + unsigned long nr_oid_total, + struct string_list *result_list) +{ + struct oidset_iter iter; + struct strbuf err404 = STRBUF_INIT; + const struct object_id *oid; + unsigned long k; + int had_404 = 0; + + gh__response_status__zero(status); + if (!nr_oid_total) + return; + + oidset_iter_init(oids, &iter); + + for (k = 0; k < nr_oid_total; k++) { + oid = oidset_iter_next(&iter); + + do__http_get__gvfs_object(status, oid, k+1, nr_oid_total, + result_list); + + /* + * If we get a 404 for an individual object, ignore + * it and get the rest. We'll fixup the 'ec' later. + */ + if (status->ec == GH__ERROR_CODE__HTTP_404) { + if (!err404.len) + strbuf_addf(&err404, "%s: from GET %s", + status->error_message.buf, + oid_to_hex(oid)); + /* + * Mark the fetch as "incomplete", but don't + * stop trying to get other chunks. + */ + had_404 = 1; + continue; + } + + if (status->ec != GH__ERROR_CODE__OK) { + /* Stop at the first hard error. */ + strbuf_addf(&status->error_message, ": from GET %s", + oid_to_hex(oid)); + goto cleanup; + } + } + +cleanup: + if (had_404 && status->ec == GH__ERROR_CODE__OK) { + strbuf_setlen(&status->error_message, 0); + strbuf_addbuf(&status->error_message, &err404); + status->ec = GH__ERROR_CODE__HTTP_404; + } + + strbuf_release(&err404); +} + +/* + * Per-thread state for a parallel POST worker. Each thread owns its + * own curl handle and index-pack child process. + */ +struct post_thread_data { + int thread_id; + CURL *curl; + struct gh__azure_throttle throttle[GH__SERVER_TYPE__NR]; + + /* Output */ + enum gh__error_code ec; + enum gh__retry_mode retry; + struct strbuf error_message; + struct string_list result_list; + int had_404; +}; + +struct post_thread_ctx { + struct post_thread_data *workers; + int nr_workers; + + /* Shared work queue: threads atomically claim blocks */ + const struct object_id * const *oid_array; + size_t nr_oids_total; + size_t block_size; + pthread_mutex_t work_mutex; + + /* + * Serialize child setup and completion. Pipe descriptors must be + * close-on-exec before another child starts, and finish_command() + * invalidates process-global path state. + */ + pthread_mutex_t spawn_mutex; + size_t next_block_start; + + /* Shared read-only state (set before threads launch) */ + const char *url; + const char *fallback_url; + const char *second_fallback_url; + struct strbuf temp_dir; + struct curl_slist *main_headers; + struct curl_slist *cache_headers; + enum gh__server_type server_type; + enum gh__server_type fallback_server_type; + int stop_requested; + + pthread_mutex_t throttle_mutex; + timestamp_t retry_after_until[GH__SERVER_TYPE__NR]; + pthread_mutex_t progress_mutex; + struct progress *progress; + int nr_finished; +}; + +struct post_response_headers { + enum gh__server_type server_type; + struct gh__azure_throttle throttle; + struct strbuf e2eid; +}; + +struct post_attempt_data { + struct post_response_headers headers; + struct strbuf ip_stdout; + struct strbuf temp_pack; + struct strbuf temp_idx; + struct strbuf final_pack; + struct strbuf final_idx; + struct strbuf final_name; +}; + +#define POST_ATTEMPT_DATA_INIT { \ + .headers = { \ + .throttle = GH__AZURE_THROTTLE_INIT, \ + .e2eid = STRBUF_INIT, \ + }, \ + .ip_stdout = STRBUF_INIT, \ + .temp_pack = STRBUF_INIT, \ + .temp_idx = STRBUF_INIT, \ + .final_pack = STRBUF_INIT, \ + .final_idx = STRBUF_INIT, \ + .final_name = STRBUF_INIT, \ +} + +static void post_attempt_data_release(struct post_attempt_data *data) +{ + strbuf_release(&data->headers.e2eid); + strbuf_release(&data->ip_stdout); + strbuf_release(&data->temp_pack); + strbuf_release(&data->temp_idx); + strbuf_release(&data->final_pack); + strbuf_release(&data->final_idx); + strbuf_release(&data->final_name); + *data = (struct post_attempt_data)POST_ATTEMPT_DATA_INIT; +} + +struct post_write_data { + int fd; + int write_error; +}; + +static size_t parse_post_response_header(char *buffer, size_t size, + size_t nitems, void *userdata) +{ + struct post_response_headers *headers = userdata; + + parse_gvfs_response_header(buffer, size, nitems, + &headers->throttle, &headers->e2eid, + headers->server_type); + + return size * nitems; +} + +static void log_post_e2eid(enum gh__server_type server_type, + enum gh__retry_mode retry, + const struct strbuf *e2eid) +{ + struct strbuf key = STRBUF_INIT; + + if (!e2eid->len || + retry == GH__RETRY_MODE__SUCCESS || + retry == GH__RETRY_MODE__HTTP_401 || + retry == GH__RETRY_MODE__FAIL_404) + return; + + strbuf_addstr(&key, "e2eid"); + strbuf_addstr(&key, gh__server_type_label[server_type]); + trace2_data_string(TR2_CAT, NULL, key.buf, e2eid->buf); + strbuf_release(&key); +} + +static struct curl_slist *build_post_headers( + const struct credential *creds) +{ + struct curl_slist *headers = http_copy_default_headers(); + + headers = curl_slist_append(headers, + "X-TFS-FedAuthRedirect: Suppress"); + headers = curl_slist_append(headers, "Pragma: no-cache"); + headers = curl_slist_append(headers, + "Content-Type: application/json"); + headers = curl_slist_append(headers, + "Accept: application/x-git-packfile"); + headers = curl_slist_append(headers, + "Accept: application/x-git-loose-object"); + append_session_id_header(&headers); + + if (creds->authtype && creds->credential) { + struct strbuf auth = STRBUF_INIT; + + strbuf_addf(&auth, "Authorization: %s %s", + creds->authtype, creds->credential); + headers = curl_slist_append(headers, auth.buf); + strbuf_release(&auth); + } + + return headers; +} + +/* + * Configure a curl handle for a gvfs/objects POST. The caller must set + * CURLOPT_WRITEFUNCTION and CURLOPT_WRITEDATA before performing the request. + */ +static void configure_post_curl_handle(CURL *curl, + struct post_thread_ctx *ctx, + enum gh__server_type server_type, + const char *url, + const char *payload, + size_t payload_len, + struct post_response_headers *headers) +{ + const struct credential *creds = + server_type == GH__SERVER_TYPE__CACHE ? + &gh__global.cache_creds : &gh__global.main_creds; + struct curl_slist *curl_headers = + server_type == GH__SERVER_TYPE__CACHE ? + ctx->cache_headers : ctx->main_headers; + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_ENCODING, NULL); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)payload_len); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_NOBODY, 0L); + /* + * Older curl versions skip response headers when FAILONERROR is + * enabled, which would hide Retry-After and authentication errors. + */ + curl_easy_setopt(curl, CURLOPT_FAILONERROR, + curl_version_info(CURLVERSION_NOW)->version_num < + 0x074b00 ? 0L : 1L); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + parse_post_response_header); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, headers); + + if (creds->authtype && creds->credential) { + /* + * Bearer token or other custom authtype from credential + * manager. Already added to the request headers by caller. + */ + } else if (creds->username) { + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_easy_setopt(curl, CURLOPT_USERNAME, + creds->username); + curl_easy_setopt(curl, CURLOPT_PASSWORD, + creds->password); + } else { + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY); + curl_easy_setopt(curl, CURLOPT_USERPWD, ":"); + } + + if (gh__global.connect_timeout_ms) + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, + gh__global.connect_timeout_ms); +} + +static void set_post_response_status(enum gh__server_type server_type, + CURLcode curl_code, + long http_response_code, + struct gh__response_status *status) +{ + struct gh__request_params params = GH__REQUEST_PARAMS_INIT; + + params.server_type = server_type; + http_response_code = gh__normalize_odd_codes(¶ms, + http_response_code); + if (http_response_code >= 400 || + curl_code == CURLE_OK || + curl_code == CURLE_HTTP_RETURNED_ERROR) + compute_retry_mode_from_http_response(status, + http_response_code); + else + compute_retry_mode_from_curl_error(status, curl_code); +} + +static void stop_post_workers(struct post_thread_ctx *ctx) +{ + pthread_mutex_lock(&ctx->work_mutex); + ctx->stop_requested = 1; + pthread_mutex_unlock(&ctx->work_mutex); +} + +static void wait_before_post_retry(enum gh__retry_mode retry, + unsigned long retry_after_sec, + int attempt) +{ + int delay_sec = 0; + + if ((retry == GH__RETRY_MODE__HTTP_429 || + retry == GH__RETRY_MODE__HTTP_503) && + !retry_after_sec) + delay_sec = compute_transient_delay(attempt); + else if (retry == GH__RETRY_MODE__TRANSIENT) + delay_sec = compute_transient_delay(attempt); + + if (delay_sec) + sleep_millisec(delay_sec * 1000); +} + +static void record_post_retry_after(struct post_thread_ctx *ctx, + enum gh__server_type server_type, + unsigned long retry_after_sec) +{ + timestamp_t now = time(NULL); + timestamp_t retry_after_until; + + if (!retry_after_sec) + return; + + if (retry_after_sec > TIME_MAX - now) + retry_after_until = TIME_MAX; + else + retry_after_until = now + retry_after_sec; + pthread_mutex_lock(&ctx->throttle_mutex); + if (ctx->retry_after_until[server_type] < retry_after_until) + ctx->retry_after_until[server_type] = retry_after_until; + pthread_mutex_unlock(&ctx->throttle_mutex); +} + +static void wait_for_post_retry_after(struct post_thread_ctx *ctx, + enum gh__server_type server_type) +{ + while (1) { + timestamp_t retry_after_until; + timestamp_t now = time(NULL); + + pthread_mutex_lock(&ctx->throttle_mutex); + retry_after_until = ctx->retry_after_until[server_type]; + pthread_mutex_unlock(&ctx->throttle_mutex); + + if (retry_after_until <= now) + return; + + sleep_millisec(100); + } +} + +static void wait_for_post_soft_throttle( + struct post_thread_data *td, enum gh__server_type server_type) +{ + struct gh__azure_throttle *throttle = &td->throttle[server_type]; + unsigned long delay_sec = throttle->reset_sec; + timestamp_t now = time(NULL); + timestamp_t end; + + /* + * Soft throttling is kept per worker rather than synchronized. POST + * parallelism is primarily used with cache servers, which are not + * expected to send Azure DevOps rate-limit headers. + */ + gh__azure_throttle__zero(throttle); + if (!delay_sec) + return; + + if (delay_sec > TIME_MAX - now) + end = TIME_MAX; + else + end = now + delay_sec; + + while (now < end) { + sleep_millisec(100); + now = time(NULL); + } +} + +struct post_thread_arg { + struct post_thread_data *td; + struct post_thread_ctx *ctx; +}; + +/* + * Curl write callback that streams data directly to a pipe fd. + */ +static size_t curl_write_to_fd(char *ptr, size_t size, size_t nmemb, + void *userdata) +{ + struct post_write_data *data = userdata; + size_t total = size * nmemb; + + if (write_in_full(data->fd, ptr, total) < 0) { + data->write_error = 1; + return 0; + } + return total; +} + +static int mark_fd_cloexec(int fd) +{ + int flags = fcntl(fd, F_GETFD); + + if (flags < 0 || fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) + return -1; + return 0; +} + +/* + * Prepare the child's stdin and stdout pipes before start_command(). Holding + * the mutex through pipe creation, CLOEXEC setup, and spawn prevents another + * child from inheriting either worker's pipe ends. + */ +static int start_post_index_pack(struct post_thread_ctx *ctx, + struct child_process *cp, + int *stdin_fd, int *stdout_fd) +{ + int in_pipe[2] = { -1, -1 }; + int out_pipe[2] = { -1, -1 }; + int ret = -1; + int saved_errno; + + pthread_mutex_lock(&ctx->spawn_mutex); + if (pipe(in_pipe) < 0 || pipe(out_pipe) < 0 || + mark_fd_cloexec(in_pipe[0]) || + mark_fd_cloexec(in_pipe[1]) || + mark_fd_cloexec(out_pipe[0]) || + mark_fd_cloexec(out_pipe[1])) + goto cleanup; + + cp->in = in_pipe[0]; + cp->out = out_pipe[1]; + in_pipe[0] = -1; + out_pipe[1] = -1; + + if (start_command(cp)) + goto cleanup; + + *stdin_fd = in_pipe[1]; + *stdout_fd = out_pipe[0]; + in_pipe[1] = -1; + out_pipe[0] = -1; + ret = 0; + +cleanup: + saved_errno = errno; + if (in_pipe[0] >= 0) + close(in_pipe[0]); + if (in_pipe[1] >= 0) + close(in_pipe[1]); + if (out_pipe[0] >= 0) + close(out_pipe[0]); + if (out_pipe[1] >= 0) + close(out_pipe[1]); + pthread_mutex_unlock(&ctx->spawn_mutex); + errno = saved_errno; + + return ret; +} + +static int finish_post_index_pack(struct post_thread_ctx *ctx, + struct child_process *cp) +{ + int ret; + + pthread_mutex_lock(&ctx->spawn_mutex); + ret = finish_command(cp); + pthread_mutex_unlock(&ctx->spawn_mutex); + return ret; +} + +static int parse_index_pack_output(struct strbuf *output, + struct object_id *pack_oid) +{ + const char *end; + + if (!skip_prefix(output->buf, "pack\t", &end) || + parse_oid_hex(end, pack_oid, &end)) + return -1; + if (*end == '\n') + end++; + return *end ? -1 : 0; +} + +static void create_post_temp_paths(struct post_thread_ctx *ctx, + int thread_id, size_t block_start, + int attempt, struct strbuf *pack_path, + struct strbuf *idx_path) +{ + strbuf_addf(pack_path, "%s/pack-%d-%"PRIuMAX"-%d.pack", + ctx->temp_dir.buf, thread_id, + (uintmax_t)block_start, attempt); + strbuf_addf(idx_path, "%s/pack-%d-%"PRIuMAX"-%d.idx", + ctx->temp_dir.buf, thread_id, + (uintmax_t)block_start, attempt); +} + +/* + * Worker thread: streams HTTP POST response directly into an + * index-pack --stdin child process, then renames the resulting + * pack-.{pack,idx} to vfs-.{pack,idx}. + */ +static void *post_worker_thread_fn(void *arg) +{ + struct post_thread_arg *a = arg; + struct post_thread_data *td = a->td; + struct post_thread_ctx *ctx = a->ctx; + + trace2_thread_start("post"); + + while (1) { + struct json_writer jw = JSON_WRITER_INIT; + struct child_process ip = CHILD_PROCESS_INIT; + struct post_attempt_data data = POST_ATTEMPT_DATA_INIT; + struct object_id pack_oid; + size_t block_start; + size_t count; + int attempt = 0; + int child_stdin = -1; + int child_stdout = -1; + CURL *curl; + CURLcode res; + long http_code = 0; + const char *request_url; + const char *fallback_url; + enum gh__server_type server_type; + enum gh__server_type fallback_server_type; + + /* Atomically claim the next block */ + pthread_mutex_lock(&ctx->work_mutex); + block_start = ctx->next_block_start; + if (ctx->stop_requested || + block_start >= ctx->nr_oids_total) { + pthread_mutex_unlock(&ctx->work_mutex); + break; + } + count = ctx->nr_oids_total - block_start; + if (count > ctx->block_size) { + if (count == ctx->block_size + 1) + count = ctx->block_size - 1; + else + count = ctx->block_size; + } + ctx->next_block_start = block_start + count; + pthread_mutex_unlock(&ctx->work_mutex); + trace2_data_intmax(TR2_CAT, NULL, "post/worker", + td->thread_id); + + request_url = ctx->url; + fallback_url = ctx->fallback_url; + server_type = ctx->server_type; + fallback_server_type = ctx->fallback_server_type; + +retry_block: + { + struct gh__response_status response = + GH__RESPONSE_STATUS_INIT; + struct post_write_data write_data = { 0 }; + + wait_for_post_retry_after(ctx, server_type); + wait_for_post_soft_throttle(td, server_type); + + child_process_init(&ip); + build_post_payload(&jw, ctx->oid_array, block_start, count); + create_post_temp_paths(ctx, td->thread_id, block_start, + attempt, &data.temp_pack, + &data.temp_idx); + + /* + * Give each child unique output paths so concurrent requests + * for the same pack cannot remove one another's source files. + */ + ip.git_cmd = 1; + strvec_push(&ip.args, "index-pack"); + strvec_push(&ip.args, "--stdin"); + strvec_push(&ip.args, "--no-rev-index"); + strvec_pushl(&ip.args, "-o", data.temp_idx.buf, NULL); + strvec_push(&ip.args, data.temp_pack.buf); + strvec_pushf(&ip.env, "GIT_OBJECT_DIRECTORY=%s", + gh__global.buf_odb_path.buf); + ip.no_stderr = 1; + + if (start_post_index_pack(ctx, &ip, &child_stdin, + &child_stdout)) { + strbuf_addf(&td->error_message, + "cannot start index-pack (worker %d): %s", + td->thread_id, strerror(errno)); + td->ec = GH__ERROR_CODE__INDEX_PACK_FAILED; + td->retry = GH__RETRY_MODE__HARD_FAIL; + jw_release(&jw); + child_process_clear(&ip); + stop_post_workers(ctx); + gh__response_status__release(&response); + post_attempt_data_release(&data); + break; + } + + write_data.fd = child_stdin; + + curl = td->curl; + data.headers.server_type = server_type; + configure_post_curl_handle(curl, ctx, server_type, + request_url, + jw.json.buf, jw.json.len, + &data.headers); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + curl_write_to_fd); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &write_data); + + http_code = 0; + trace2_region_enter_printf(TR2_CAT, "post/curl", NULL, + "worker:%d attempt:%d", + td->thread_id, attempt); + res = curl_easy_perform(curl); + trace2_region_leave(TR2_CAT, "post/curl", NULL); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, + &http_code); + td->throttle[server_type] = data.headers.throttle; + + close(write_data.fd); + child_stdin = -1; + jw_release(&jw); + + set_post_response_status(server_type, res, http_code, + &response); + log_post_e2eid(server_type, response.retry, + &data.headers.e2eid); + if (response.retry == GH__RETRY_MODE__HTTP_429 || + response.retry == GH__RETRY_MODE__HTTP_503) + record_post_retry_after(ctx, server_type, + data.headers.throttle.retry_after_sec); + + if (response.retry != GH__RETRY_MODE__SUCCESS) { + close(child_stdout); + child_stdout = -1; + finish_post_index_pack(ctx, &ip); + child_process_clear(&ip); + unlink(data.temp_pack.buf); + unlink(data.temp_idx.buf); + + if ((response.retry == GH__RETRY_MODE__TRANSIENT || + response.retry == GH__RETRY_MODE__HTTP_429 || + response.retry == GH__RETRY_MODE__HTTP_503) && + attempt < gh__cmd_opts.max_retries) { + wait_before_post_retry( + response.retry, + data.headers.throttle.retry_after_sec, + attempt); + attempt++; + gh__response_status__release(&response); + post_attempt_data_release(&data); + goto retry_block; + } + + if (fallback_url && + (response.retry != GH__RETRY_MODE__HTTP_401 || + fallback_server_type == + GH__SERVER_TYPE__CACHE)) { + request_url = fallback_url; + server_type = fallback_server_type; + if (fallback_url == ctx->fallback_url && + ctx->second_fallback_url) { + fallback_url = + ctx->second_fallback_url; + fallback_server_type = + GH__SERVER_TYPE__MAIN; + } else { + fallback_url = NULL; + } + attempt = 0; + gh__response_status__release(&response); + post_attempt_data_release(&data); + goto retry_block; + } + + if (response.retry == GH__RETRY_MODE__FAIL_404) { + td->had_404 = 1; + if (!td->error_message.len) + strbuf_addbuf(&td->error_message, + &response.error_message); + pthread_mutex_lock(&ctx->progress_mutex); + ctx->nr_finished++; + display_progress(ctx->progress, + ctx->nr_finished); + pthread_mutex_unlock(&ctx->progress_mutex); + gh__response_status__release(&response); + post_attempt_data_release(&data); + continue; + } + + td->ec = response.ec; + td->retry = response.retry; + if (td->error_message.len) + strbuf_addstr(&td->error_message, "; "); + strbuf_addbuf(&td->error_message, + &response.error_message); + strbuf_addstr(&td->error_message, ": from POST"); + stop_post_workers(ctx); + gh__response_status__release(&response); + post_attempt_data_release(&data); + break; + } + + /* + * Read index-pack stdout and wait for exit. + * With --stdin, format is: "pack\t\n" + */ + strbuf_read(&data.ip_stdout, child_stdout, 128); + close(child_stdout); + child_stdout = -1; + + if (finish_post_index_pack(ctx, &ip) || + parse_index_pack_output(&data.ip_stdout, &pack_oid)) { + unlink(data.temp_pack.buf); + unlink(data.temp_idx.buf); + log_post_e2eid(server_type, + GH__RETRY_MODE__TRANSIENT, + &data.headers.e2eid); + child_process_clear(&ip); + gh__response_status__release(&response); + + if (attempt < gh__cmd_opts.max_retries) { + wait_before_post_retry( + GH__RETRY_MODE__TRANSIENT, 0, + attempt); + attempt++; + post_attempt_data_release(&data); + goto retry_block; + } + + if (fallback_url) { + request_url = fallback_url; + server_type = fallback_server_type; + if (fallback_url == ctx->fallback_url && + ctx->second_fallback_url) { + fallback_url = + ctx->second_fallback_url; + fallback_server_type = + GH__SERVER_TYPE__MAIN; + } else { + fallback_url = NULL; + } + attempt = 0; + post_attempt_data_release(&data); + goto retry_block; + } + + strbuf_addf(&td->error_message, + "index-pack failed (worker %d)", + td->thread_id); + td->ec = GH__ERROR_CODE__INDEX_PACK_FAILED; + td->retry = GH__RETRY_MODE__HARD_FAIL; + stop_post_workers(ctx); + post_attempt_data_release(&data); + break; + } + child_process_clear(&ip); + gh__response_status__release(&response); + + { + char hash_hex[GIT_MAX_HEXSZ + 1]; + + oid_to_hex_r(hash_hex, &pack_oid); + + create_final_packfile_pathnames( + "vfs", hash_hex, NULL, + &data.final_pack, &data.final_idx, + &data.final_name); + + if (my_finalize_packfile_simple( + data.temp_pack.buf, data.temp_idx.buf, + data.final_pack.buf, + data.final_idx.buf)) { + strbuf_addf(&td->error_message, + "could not install packfile %s", + data.final_name.buf); + td->ec = + GH__ERROR_CODE__INDEX_PACK_FAILED; + td->retry = GH__RETRY_MODE__HARD_FAIL; + } + } + + if (td->ec != GH__ERROR_CODE__OK) { + stop_post_workers(ctx); + post_attempt_data_release(&data); + break; + } + + /* Record result */ + { + struct strbuf msg = STRBUF_INIT; + strbuf_addf(&msg, "packfile %s", + data.final_name.buf); + string_list_append(&td->result_list, msg.buf); + strbuf_release(&msg); + } + + /* Update progress under mutex */ + pthread_mutex_lock(&ctx->progress_mutex); + ctx->nr_finished++; + display_progress(ctx->progress, ctx->nr_finished); + pthread_mutex_unlock(&ctx->progress_mutex); + + post_attempt_data_release(&data); + } + } + + trace2_thread_exit(); + return NULL; +} + +static int create_post_temp_dir(struct post_thread_ctx *ctx, + struct gh__response_status *status) +{ + enum scld_error scld; + + strbuf_addbuf(&ctx->temp_dir, &gh__global.buf_odb_path); + strbuf_complete(&ctx->temp_dir, '/'); + strbuf_addstr(&ctx->temp_dir, "pack/tempPacks/post-XXXXXX"); + + scld = safe_create_leading_directories(the_repository, + ctx->temp_dir.buf); + if (scld != SCLD_OK && scld != SCLD_EXISTS) + goto error; + if (!mkdtemp(ctx->temp_dir.buf)) + goto error; + return 0; + +error: + strbuf_addf(&status->error_message, + "could not create directory for POST packfiles: '%s'", + ctx->temp_dir.buf); + status->ec = GH__ERROR_CODE__COULD_NOT_CREATE_TEMPFILE; + status->retry = GH__RETRY_MODE__HARD_FAIL; + return -1; +} + +static int should_use_parallel_post(size_t nr_oids) +{ + if (!HAVE_THREADS || gh__global.post_threads <= 1 || + gh__cmd_opts.block_size < + GH__MIN_OBJECTS_POST__PARALLEL_BLOCK_SIZE || + nr_oids <= 1 || + http_cookies_configured()) + return 0; + + return 1; +} + +/* + * Drive one or more HTTP POST requests to bulk fetch the objects in + * the given OIDSET. Create one or more packfiles and/or loose objects. + * + * Accumulate results for each request in `result_list` until we get a + * hard error and have to stop. + */ +static void do__http_post__fetch_oidset(struct gh__response_status *status, + struct oidset *oids, + unsigned long nr_oid_total, + struct string_list *result_list) +{ + struct oidset_iter iter; + struct strbuf err404 = STRBUF_INIT; + unsigned long k; + unsigned long nr_oid_taken; + int j_pack_den = 0; + int j_pack_num = 0; + int had_404 = 0; + int use_threaded; + + gh__response_status__zero(status); + if (!nr_oid_total) + return; + + use_threaded = should_use_parallel_post(nr_oid_total); + trace2_data_intmax(TR2_CAT, NULL, "post/fetch_mode", + use_threaded ? gh__global.post_threads : 1); + + if (use_threaded) { + const struct object_id *oid; + const struct object_id **oid_array; + int nr_workers; + struct post_thread_ctx ctx; + struct post_thread_arg *args; + pthread_t *threads; + struct strbuf url = STRBUF_INIT; + size_t nr_batches; + int nr_started = 0; + int auth_retries = 0; + int i; + enum gh__server_type initial_server_type; + + update_cache_server_for_verb(POST); + initial_server_type = gh__global.cache_server_url ? + GH__SERVER_TYPE__CACHE : GH__SERVER_TYPE__MAIN; + + /* + * Cache servers require pre-filled Basic credentials. + * Main servers start with CURLAUTH_ANY so libcurl can + * negotiate authentication before we fill credentials. + */ + if (initial_server_type == GH__SERVER_TYPE__CACHE) + synthesize_cache_server_creds(); + +retry_threaded: + gh__response_status__zero(status); + had_404 = 0; + nr_started = 0; + + /* Collect pointers to the oidset entries. */ + ALLOC_ARRAY(oid_array, nr_oid_total); + oidset_iter_init(oids, &iter); + for (k = 0; (oid = oidset_iter_next(&iter)); k++) + oid_array[k] = oid; + + nr_batches = nr_oid_total / gh__cmd_opts.block_size + + !!(nr_oid_total % gh__cmd_opts.block_size); + nr_workers = nr_batches < + (size_t)gh__global.post_threads ? + (int)nr_batches : gh__global.post_threads; + + /* Build URL (and fallback for cache-server mode) */ + { + struct strbuf fallback = STRBUF_INIT; + struct strbuf second_fallback = STRBUF_INIT; + + if (gh__global.cache_server_url) { + end_url_with_slash(&url, + gh__global.cache_server_url); + if (gh__cmd_opts.try_fallback) { + const char *backup = + gh__global + .cache_server_url_backup; + + if (backup) { + end_url_with_slash(&fallback, + backup); + strbuf_addstr(&fallback, + "gvfs/objects"); + end_url_with_slash( + &second_fallback, + gh__global.main_url); + strbuf_addstr(&second_fallback, + "gvfs/objects"); + } else { + end_url_with_slash(&fallback, + gh__global.main_url); + strbuf_addstr(&fallback, + "gvfs/objects"); + } + } + } else { + end_url_with_slash(&url, + gh__global.main_url); + } + strbuf_addstr(&url, "gvfs/objects"); + + memset(&ctx, 0, sizeof(ctx)); + strbuf_init(&ctx.temp_dir, 0); + ctx.url = url.buf; + if (fallback.len) + ctx.fallback_url = strbuf_detach( + &fallback, NULL); + else + strbuf_release(&fallback); + if (second_fallback.len) + ctx.second_fallback_url = strbuf_detach( + &second_fallback, NULL); + else + strbuf_release(&second_fallback); + } + + ctx.server_type = initial_server_type; + ctx.fallback_server_type = + ctx.second_fallback_url ? + GH__SERVER_TYPE__CACHE : GH__SERVER_TYPE__MAIN; + if (create_post_temp_dir(&ctx, status)) { + free((char *)ctx.fallback_url); + free((char *)ctx.second_fallback_url); + strbuf_release(&ctx.temp_dir); + strbuf_release(&url); + free(oid_array); + reset_cache_server(); + return; + } + ctx.main_headers = build_post_headers( + &gh__global.main_creds); + ctx.cache_headers = build_post_headers( + &gh__global.cache_creds); + + ctx.nr_workers = nr_workers; + ctx.nr_finished = 0; + pthread_mutex_init(&ctx.progress_mutex, NULL); + + /* Shared work queue */ + ctx.oid_array = oid_array; + ctx.nr_oids_total = nr_oid_total; + ctx.block_size = gh__cmd_opts.block_size; + ctx.next_block_start = 0; + pthread_mutex_init(&ctx.work_mutex, NULL); + pthread_mutex_init(&ctx.spawn_mutex, NULL); + pthread_mutex_init(&ctx.throttle_mutex, NULL); + + if (gh__cmd_opts.show_progress) { + int total_blocks = (int)((nr_oid_total + + gh__cmd_opts.block_size - 1) / + gh__cmd_opts.block_size); + ctx.progress = start_progress( + the_repository, + "Fetching objects (parallel)", + total_blocks); + } + + /* Allocate per-worker state */ + CALLOC_ARRAY(ctx.workers, nr_workers); + ALLOC_ARRAY(args, nr_workers); + ALLOC_ARRAY(threads, nr_workers); + + for (i = 0; i < nr_workers; i++) { + ctx.workers[i].thread_id = i; + ctx.workers[i].curl = http_get_curl_handle(); + ctx.workers[i].ec = GH__ERROR_CODE__OK; + ctx.workers[i].retry = + GH__RETRY_MODE__SUCCESS; + strbuf_init(&ctx.workers[i].error_message, 0); + ctx.workers[i].result_list.strdup_strings = 1; + ctx.workers[i].had_404 = 0; + + args[i].td = &ctx.workers[i]; + args[i].ctx = &ctx; + } + + sigchain_push(SIGPIPE, SIG_IGN); + + /* Spawn threads */ + for (i = 0; i < nr_workers; i++) { + if (pthread_create(&threads[i], NULL, + post_worker_thread_fn, + &args[i])) { + strbuf_addf(&status->error_message, + "pthread_create failed for " + "worker %d", i); + status->ec = + GH__ERROR_CODE__INDEX_PACK_FAILED; + status->retry = + GH__RETRY_MODE__HARD_FAIL; + break; + } + nr_started++; + } + + /* Wait for all threads */ + for (i = 0; i < nr_started; i++) + pthread_join(threads[i], NULL); + + /* Collect results */ + for (i = 0; i < nr_workers; i++) { + if (ctx.workers[i].had_404) + had_404 = 1; + + if (ctx.workers[i].ec != GH__ERROR_CODE__OK && + status->ec == GH__ERROR_CODE__OK) { + status->ec = ctx.workers[i].ec; + status->retry = ctx.workers[i].retry; + strbuf_addbuf(&status->error_message, + &ctx.workers[i].error_message); + } + } + + for (i = 0; + status->retry != GH__RETRY_MODE__HTTP_401 && + i < nr_workers; + i++) { + size_t j; + struct string_list *wrl = + &ctx.workers[i].result_list; + + for (j = 0; j < wrl->nr; j++) + string_list_append(result_list, + wrl->items[j].string); + } + + if (had_404 && status->ec == GH__ERROR_CODE__OK) { + for (i = 0; i < nr_workers; i++) { + if (ctx.workers[i].had_404) { + strbuf_addbuf(&status->error_message, + &ctx.workers[i].error_message); + break; + } + } + status->ec = GH__ERROR_CODE__HTTP_404; + status->retry = GH__RETRY_MODE__FAIL_404; + } + + stop_progress(&ctx.progress); + pthread_mutex_destroy(&ctx.progress_mutex); + pthread_mutex_destroy(&ctx.work_mutex); + pthread_mutex_destroy(&ctx.spawn_mutex); + pthread_mutex_destroy(&ctx.throttle_mutex); + curl_slist_free_all(ctx.main_headers); + curl_slist_free_all(ctx.cache_headers); + free((char *)ctx.fallback_url); + free((char *)ctx.second_fallback_url); + remove_dir_recursively(&ctx.temp_dir, 0); + strbuf_release(&ctx.temp_dir); + strbuf_release(&url); + for (i = 0; i < nr_workers; i++) { + if (ctx.workers[i].curl) + curl_easy_cleanup(ctx.workers[i].curl); + strbuf_release(&ctx.workers[i].error_message); + string_list_clear( + &ctx.workers[i].result_list, 0); + } + free(ctx.workers); + free(args); + free(threads); + free(oid_array); + sigchain_pop(SIGPIPE); + + if (status->retry == GH__RETRY_MODE__HTTP_401 && + !auth_retries) { + auth_retries++; + trace2_data_intmax(TR2_CAT, NULL, + "post/auth_retry", + auth_retries); + if (initial_server_type == GH__SERVER_TYPE__CACHE) + refresh_cache_server_creds(); + else + refresh_main_creds(); + goto retry_threaded; + } + + if (status->ec == GH__ERROR_CODE__OK) { + if (initial_server_type == GH__SERVER_TYPE__CACHE) + approve_cache_server_creds(); + else + approve_main_creds(); + } + reset_cache_server(); + return; + } + + oidset_iter_init(oids, &iter); + + j_pack_den = ((nr_oid_total + gh__cmd_opts.block_size - 1) + / gh__cmd_opts.block_size); + + for (k = 0; k < nr_oid_total; k += nr_oid_taken) { + j_pack_num++; + + do__http_post__gvfs_objects(status, &iter, + gh__cmd_opts.block_size, + j_pack_num, j_pack_den, + result_list, + &nr_oid_taken); + + /* + * Because the oidset iterator has random + * order, it does no good to say the k-th or + * n-th chunk was incomplete; the client + * cannot use that index for anything. + * + * We get a 404 when at least one object in + * the chunk was not found. + * + * For now, ignore the 404 and go on to the + * next chunk and then fixup the 'ec' later. + */ + if (status->ec == GH__ERROR_CODE__HTTP_404) { + if (!err404.len) + strbuf_addf(&err404, + "%s: from POST", + status->error_message.buf); + /* + * Mark the fetch as "incomplete", but don't + * stop trying to get other chunks. + */ + had_404 = 1; + continue; + } + + if (status->ec != GH__ERROR_CODE__OK) { + /* Stop at the first hard error. */ + strbuf_addstr(&status->error_message, + ": from POST"); + goto cleanup; + } + } + +cleanup: + if (had_404 && status->ec == GH__ERROR_CODE__OK) { + strbuf_setlen(&status->error_message, 0); + strbuf_addbuf(&status->error_message, &err404); + status->ec = GH__ERROR_CODE__HTTP_404; + } + + strbuf_release(&err404); +} + +/* + * Finish with initialization. This happens after the main option + * parsing, dispatch to sub-command, and sub-command option parsing + * and before actually doing anything. + * + * Optionally configure the cache-server if the sub-command will + * use it. + */ +static void finish_init(int setup_cache_server) +{ + select_odb(); + + lookup_main_url(); + gh_http_init(); + + if (setup_cache_server) + select_cache_server(); +} + +/* + * Request gvfs/config from main Git server. (Config data is not + * available from a GVFS cache-server.) + * + * Print the received server configuration (as the raw JSON string). + */ +static enum gh__error_code do_sub_cmd__config(int argc UNUSED, const char **argv UNUSED) +{ + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct strbuf config_data = STRBUF_INIT; + enum gh__error_code ec = GH__ERROR_CODE__OK; + + trace2_cmd_mode("config"); + + finish_init(0); + + do__http_get__gvfs_config(&status, &config_data); + ec = status.ec; + + if (ec == GH__ERROR_CODE__OK) + printf("%s\n", config_data.buf); + else + error("config: %s", status.error_message.buf); + + gh__response_status__release(&status); + strbuf_release(&config_data); + + return ec; +} + +static enum gh__error_code do_sub_cmd__endpoint(int argc, const char **argv) +{ + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct strbuf data = STRBUF_INIT; + enum gh__error_code ec = GH__ERROR_CODE__OK; + const char *endpoint; + + if (argc != 2) + return GH__ERROR_CODE__ERROR; + endpoint = argv[1]; + + trace2_cmd_mode(endpoint); + + finish_init(0); + + do__http_get__simple_endpoint(&status, &data, endpoint, endpoint); + ec = status.ec; + + if (ec == GH__ERROR_CODE__OK) + printf("%s\n", data.buf); + else + error("config: %s", status.error_message.buf); + + gh__response_status__release(&status); + strbuf_release(&data); + + return ec; +} + +/* + * Read a list of objects from stdin and fetch them as a series of + * single object HTTP GET requests. + */ +static enum gh__error_code do_sub_cmd__get(int argc, const char **argv) +{ + static struct option get_options[] = { + OPT_INTEGER('r', "max-retries", &gh__cmd_opts.max_retries, + N_("retries for transient network errors")), + OPT_UNSIGNED(0, "connect-timeout-ms", + &gh__global.connect_timeout_ms, + N_("try to connect only for this many milliseconds")), + OPT_END(), + }; + + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct oidset oids = OIDSET_INIT; + struct string_list result_list = STRING_LIST_INIT_DUP; + enum gh__error_code ec = GH__ERROR_CODE__OK; + unsigned long nr_oid_total; + size_t k; + + trace2_cmd_mode("get"); + + if (argc > 1 && !strcmp(argv[1], "-h")) + usage_with_options(objects_get_usage, get_options); + + argc = parse_options(argc, argv, NULL, get_options, objects_get_usage, 0); + if (gh__cmd_opts.max_retries < 0) + gh__cmd_opts.max_retries = 0; + + finish_init(1); + + nr_oid_total = read_stdin_for_oids(&oids); + + do__http_get__fetch_oidset(&status, &oids, nr_oid_total, &result_list); + + ec = status.ec; + + for (k = 0; k < result_list.nr; k++) + printf("%s\n", result_list.items[k].string); + + if (ec != GH__ERROR_CODE__OK) + error("get: %s", status.error_message.buf); + + gh__response_status__release(&status); + oidset_clear(&oids); + string_list_clear(&result_list, 0); + + return ec; +} + +/* + * Read a list of objects from stdin and fetch them in a single request (or + * multiple block-size requests) using one or more HTTP POST requests. + */ +static enum gh__error_code do_sub_cmd__post(int argc, const char **argv) +{ + static struct option post_options[] = { + OPT_UNSIGNED('b', "block-size", &gh__cmd_opts.block_size, + N_("number of objects to request at a time")), + OPT_INTEGER('d', "depth", &gh__cmd_opts.depth, + N_("Commit depth")), + OPT_INTEGER('r', "max-retries", &gh__cmd_opts.max_retries, + N_("retries for transient network errors")), + OPT_END(), + }; + + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct oidset oids = OIDSET_INIT; + struct string_list result_list = STRING_LIST_INIT_DUP; + enum gh__error_code ec = GH__ERROR_CODE__OK; + unsigned long nr_oid_total; + size_t k; + + trace2_cmd_mode("post"); + + if (argc > 1 && !strcmp(argv[1], "-h")) + usage_with_options(objects_post_usage, post_options); + + argc = parse_options(argc, argv, NULL, post_options, objects_post_usage, 0); + if (gh__cmd_opts.depth < 1) + gh__cmd_opts.depth = 1; + if (gh__cmd_opts.max_retries < 0) + gh__cmd_opts.max_retries = 0; + + finish_init(1); + + nr_oid_total = read_stdin_for_oids(&oids); + + do__http_post__fetch_oidset(&status, &oids, nr_oid_total, &result_list); + + ec = status.ec; + + for (k = 0; k < result_list.nr; k++) + printf("%s\n", result_list.items[k].string); + + if (ec != GH__ERROR_CODE__OK) + error("post: %s", status.error_message.buf); + + gh__response_status__release(&status); + oidset_clear(&oids); + string_list_clear(&result_list, 0); + + return ec; +} + +/* + * Interpret the given string as a timestamp and compute an absolute + * UTC-seconds-since-epoch value (and without TZ). + * + * Note that the gvfs/prefetch API only accepts seconds since epoch, + * so that is all we really need here. But there is a tradition of + * various Git commands allowing a variety of formats for args like + * this. For example, see the `--date` arg in `git commit`. We allow + * these other forms mainly for testing purposes. + */ +static int my_parse_since(const char *since, timestamp_t *p_timestamp) +{ + int offset = 0; + int errors = 0; + unsigned long t; + + if (!parse_date_basic(since, p_timestamp, &offset)) + return 0; + + t = approxidate_careful(since, &errors); + if (!errors) { + *p_timestamp = t; + return 0; + } + + return -1; +} + +/* + * Ask the server for all available packfiles -or- all available since + * the given timestamp. + */ +static enum gh__error_code do_sub_cmd__prefetch(int argc, const char **argv) +{ + static const char *since_str; + static struct option prefetch_options[] = { + OPT_STRING(0, "since", &since_str, N_("since"), N_("seconds since epoch")), + OPT_INTEGER('r', "max-retries", &gh__cmd_opts.max_retries, + N_("retries for transient network errors")), + OPT_END(), + }; + + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct string_list result_list = STRING_LIST_INIT_DUP; + enum gh__error_code ec = GH__ERROR_CODE__OK; + timestamp_t seconds_since_epoch = 0; + size_t k; + + trace2_cmd_mode("prefetch"); + + if (argc > 1 && !strcmp(argv[1], "-h")) + usage_with_options(prefetch_usage, prefetch_options); + + argc = parse_options(argc, argv, NULL, prefetch_options, prefetch_usage, 0); + if (since_str && *since_str) { + if (my_parse_since(since_str, &seconds_since_epoch)) + die("could not parse 'since' field"); + } + if (gh__cmd_opts.max_retries < 0) + gh__cmd_opts.max_retries = 0; + + finish_init(1); + + do__http_get__gvfs_prefetch(&status, seconds_since_epoch, &result_list); + + ec = status.ec; + + for (k = 0; k < result_list.nr; k++) + printf("%s\n", result_list.items[k].string); + + if (ec != GH__ERROR_CODE__OK) + error("prefetch: %s", status.error_message.buf); + + gh__response_status__release(&status); + string_list_clear(&result_list, 0); + + return ec; +} + +/* + * Handle the 'objects.get' and 'objects.post' and 'objects.prefetch' + * verbs in "server mode". + * + * Only call error() and set ec for hard errors where we cannot + * communicate correctly with the foreground client process. Pass any + * actual data errors (such as 404's or 401's from the fetch) back to + * the client process. + */ +static enum gh__error_code do_server_subprocess__objects(const char *verb_line) +{ + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct oidset oids = OIDSET_INIT; + struct object_id oid; + struct string_list result_list = STRING_LIST_INIT_DUP; + enum gh__error_code ec = GH__ERROR_CODE__OK; + char *line; + int len; + int err; + size_t k; + enum gh__objects_mode objects_mode; + unsigned long nr_oid_total = 0; + timestamp_t seconds_since_epoch = 0; + + if (!strcmp(verb_line, "objects.get")) + objects_mode = GH__OBJECTS_MODE__GET; + else if (!strcmp(verb_line, "objects.post")) + objects_mode = GH__OBJECTS_MODE__POST; + else if (!strcmp(verb_line, "objects.prefetch")) + objects_mode = GH__OBJECTS_MODE__PREFETCH; + else { + error("server: unexpected objects-mode verb '%s'", verb_line); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + + switch (objects_mode) { + case GH__OBJECTS_MODE__GET: + case GH__OBJECTS_MODE__POST: + while (1) { + len = packet_read_line_gently(0, NULL, &line); + if (len < 0 || !line) + break; + + if (get_oid_hex(line, &oid)) { + error("server: invalid oid syntax '%s'", line); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + + if (!oidset_insert(&oids, &oid)) + nr_oid_total++; + } + + if (!nr_oid_total) { + /* if zero objects requested, trivial OK. */ + if (packet_write_fmt_gently(1, "ok\n")) { + error("server: cannot write 'get' result to client"); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + } else + ec = GH__ERROR_CODE__OK; + goto cleanup; + } + + if (objects_mode == GH__OBJECTS_MODE__GET) + do__http_get__fetch_oidset(&status, &oids, + nr_oid_total, &result_list); + else + do__http_post__fetch_oidset(&status, &oids, + nr_oid_total, &result_list); + break; + + case GH__OBJECTS_MODE__PREFETCH: + /* get optional timestamp line */ + while (1) { + len = packet_read_line_gently(0, NULL, &line); + if (len < 0 || !line) + break; + + seconds_since_epoch = strtoul(line, NULL, 10); + } + + do__http_get__gvfs_prefetch(&status, seconds_since_epoch, + &result_list); + break; + + default: + BUG("unexpected object_mode in switch '%d'", objects_mode); + } + + /* + * Write pathname of the ODB where we wrote all of the objects + * we fetched. + */ + if (packet_write_fmt_gently(1, "odb %s\n", + gh__global.buf_odb_path.buf)) { + error("server: cannot write 'odb' to client"); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + + for (k = 0; k < result_list.nr; k++) + if (packet_write_fmt_gently(1, "%s\n", + result_list.items[k].string)) + { + error("server: cannot write result to client: '%s'", + result_list.items[k].string); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + + /* + * We only use status.ec to tell the client whether the request + * was complete, incomplete, or had IO errors. We DO NOT return + * this value to our caller. + */ + err = 0; + if (status.ec == GH__ERROR_CODE__OK) + err = packet_write_fmt_gently(1, "ok\n"); + else if (status.ec == GH__ERROR_CODE__HTTP_404) + err = packet_write_fmt_gently(1, "partial\n"); + else + err = packet_write_fmt_gently(1, "error %s\n", + status.error_message.buf); + if (err) { + error("server: cannot write result to client"); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + + if (packet_flush_gently(1)) { + error("server: cannot flush result to client"); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + +cleanup: + oidset_clear(&oids); + string_list_clear(&result_list, 0); + gh__response_status__release(&status); + + return ec; +} + +typedef enum gh__error_code (fn_subprocess_cmd)(const char *verb_line); + +struct subprocess_capability { + const char *name; + int client_has; + fn_subprocess_cmd *pfn; +}; + +static struct subprocess_capability caps[] = { + { "objects", 0, do_server_subprocess__objects }, + { NULL, 0, NULL }, +}; + +/* + * Handle the subprocess protocol handshake as described in: + * [] Documentation/technical/protocol-common.txt + * [] Documentation/technical/long-running-process-protocol.txt + */ +static int do_protocol_handshake(void) +{ +#define OUR_SUBPROCESS_VERSION "1" + + char *line; + int len; + int k; + int b_support_our_version = 0; + + len = packet_read_line_gently(0, NULL, &line); + if (len < 0 || !line || strcmp(line, "gvfs-helper-client")) { + error("server: subprocess welcome handshake failed: %s", line); + return -1; + } + + while (1) { + const char *v; + len = packet_read_line_gently(0, NULL, &line); + if (len < 0 || !line) + break; + if (!skip_prefix(line, "version=", &v)) { + error("server: subprocess version handshake failed: %s", + line); + return -1; + } + b_support_our_version |= (!strcmp(v, OUR_SUBPROCESS_VERSION)); + } + if (!b_support_our_version) { + error("server: client does not support our version: %s", + OUR_SUBPROCESS_VERSION); + return -1; + } + + if (packet_write_fmt_gently(1, "gvfs-helper-server\n") || + packet_write_fmt_gently(1, "version=%s\n", + OUR_SUBPROCESS_VERSION) || + packet_flush_gently(1)) { + error("server: cannot write version handshake"); + return -1; + } + + while (1) { + const char *v; + int k; + + len = packet_read_line_gently(0, NULL, &line); + if (len < 0 || !line) + break; + if (!skip_prefix(line, "capability=", &v)) { + error("server: subprocess capability handshake failed: %s", + line); + return -1; + } + for (k = 0; caps[k].name; k++) + if (!strcmp(v, caps[k].name)) + caps[k].client_has = 1; + } + + for (k = 0; caps[k].name; k++) + if (caps[k].client_has) + if (packet_write_fmt_gently(1, "capability=%s\n", + caps[k].name)) { + error("server: cannot write capabilities handshake: %s", + caps[k].name); + return -1; + } + if (packet_flush_gently(1)) { + error("server: cannot write capabilities handshake"); + return -1; + } + + return 0; +} + +/* + * Interactively listen to stdin for a series of commands and execute them. + */ +static enum gh__error_code do_sub_cmd__server(int argc, const char **argv) +{ + static struct option server_options[] = { + OPT_UNSIGNED('b', "block-size", &gh__cmd_opts.block_size, + N_("number of objects to request at a time")), + OPT_INTEGER('d', "depth", &gh__cmd_opts.depth, + N_("Commit depth")), + OPT_INTEGER('r', "max-retries", &gh__cmd_opts.max_retries, + N_("retries for transient network errors")), + OPT_END(), + }; + + enum gh__error_code ec = GH__ERROR_CODE__OK; + char *line; + int len; + int k; + + trace2_cmd_mode("server"); + + if (argc > 1 && !strcmp(argv[1], "-h")) + usage_with_options(server_usage, server_options); + + argc = parse_options(argc, argv, NULL, server_options, server_usage, 0); + if (gh__cmd_opts.depth < 1) + gh__cmd_opts.depth = 1; + if (gh__cmd_opts.max_retries < 0) + gh__cmd_opts.max_retries = 0; + + finish_init(1); + + if (do_protocol_handshake()) { + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + +top_of_loop: + while (1) { + len = packet_read_line_gently(0, NULL, &line); + if (len < 0 || !line) { + /* use extra FLUSH as a QUIT */ + ec = GH__ERROR_CODE__OK; + goto cleanup; + } + + for (k = 0; caps[k].name; k++) { + if (caps[k].client_has && + starts_with(line, caps[k].name)) { + ec = (caps[k].pfn)(line); + if (ec != GH__ERROR_CODE__OK) + goto cleanup; + goto top_of_loop; + } + } + + error("server: unknown command '%s'", line); + ec = GH__ERROR_CODE__SUBPROCESS_SYNTAX; + goto cleanup; + } + +cleanup: + return ec; +} + +static enum gh__error_code do_sub_cmd__curl_version(int argc, const char **argv) +{ + static struct option curl_version_options[] = { + OPT_END(), + }; + const char *current_version = curl_version_info(CURLVERSION_NOW)->version; + + trace2_cmd_mode("curl-version"); + + if (argc > 1 && !strcmp(argv[1], "-h")) + usage_with_options(curl_version_usage, curl_version_options); + + argc = parse_options(argc, argv, NULL, + curl_version_options, curl_version_usage, 0); + + if (argc == 0) + printf("%s\n", current_version); + else if (argc != 2) + die("expected [ ], but got %d parameters", argc); + else { + int cmp = versioncmp(current_version, argv[1]); + + return (strchr(argv[0], '=') && !cmp) || + (strchr(argv[0], '>') && cmp > 0) || + (strchr(argv[0], '<') && cmp < 0) ? + GH__ERROR_CODE__OK : GH__ERROR_CODE__ERROR; + } + + return GH__ERROR_CODE__OK; +} + +static enum gh__error_code do_sub_cmd(int argc, const char **argv) +{ + if (!strcmp(argv[0], "get")) + return do_sub_cmd__get(argc, argv); + + if (!strcmp(argv[0], "post")) + return do_sub_cmd__post(argc, argv); + + if (!strcmp(argv[0], "config")) + return do_sub_cmd__config(argc, argv); + + if (!strcmp(argv[0], "endpoint")) + return do_sub_cmd__endpoint(argc, argv); + + if (!strcmp(argv[0], "prefetch")) + return do_sub_cmd__prefetch(argc, argv); + + /* + * server mode is for talking with git.exe via the "gh_client_" API + * using packet-line format. + */ + if (!strcmp(argv[0], "server")) + return do_sub_cmd__server(argc, argv); + + if (!strcmp(argv[0], "curl-version")) + return do_sub_cmd__curl_version(argc, argv); + + return GH__ERROR_CODE__USAGE; +} + +/* + * Communicate with the primary Git server or a GVFS cache-server using the + * GVFS Protocol. + * + * https://github.com/microsoft/VFSForGit/blob/master/Protocol.md + */ +int cmd_main(int argc, const char **argv) +{ + static struct option main_options[] = { + OPT_STRING('r', "remote", &gh__cmd_opts.remote_name, + N_("remote"), + N_("Remote name")), + OPT_BOOL('f', "fallback", &gh__cmd_opts.try_fallback, + N_("Fallback to Git server if cache-server fails")), + OPT_CALLBACK(0, "cache-server", NULL, + N_("cache-server"), + N_("cache-server=disable|trust|verify|error"), + option_parse_cache_server_mode), + OPT_CALLBACK(0, "shared-cache", NULL, + N_("pathname"), + N_("Pathname to shared objects directory"), + option_parse_shared_cache_directory), + OPT_BOOL('p', "progress", &gh__cmd_opts.show_progress, + N_("Show progress")), + OPT_END(), + }; + + enum gh__error_code ec = GH__ERROR_CODE__OK; + + if (argc > 1 && !strcmp(argv[1], "-h")) + usage_with_options(main_usage, main_options); + + trace2_cmd_name("gvfs-helper"); + packet_trace_identity("gvfs-helper"); + + setup_git_directory_gently(the_repository, NULL); + + /* Set any non-zero initial values in gh__cmd_opts. */ + gh__cmd_opts.depth = GH__DEFAULT__OBJECTS_POST__COMMIT_DEPTH; + gh__cmd_opts.block_size = GH__DEFAULT__OBJECTS_POST__BLOCK_SIZE; + gh__cmd_opts.max_retries = GH__DEFAULT_MAX_RETRIES; + gh__cmd_opts.max_transient_backoff_sec = + GH__DEFAULT_MAX_TRANSIENT_BACKOFF_SEC; + + gh__cmd_opts.show_progress = !!isatty(2); + + // TODO use existing gvfs config settings to override our GH__DEFAULT_ + // TODO values in gh__cmd_opts. (And maybe add/remove our command line + // TODO options for them.) + // TODO + // TODO See "scalar.max-retries" (and maybe "gvfs.max-retries") + + repo_config(the_repository, git_default_config, NULL); + + /* + * Read gvfs.prefetchThreads to control parallel index-pack + * during prefetch. Default to 1 (sequential) for safety. + */ + gh__global.prefetch_threads = 1; + repo_config_get_int(the_repository, "gvfs.prefetchthreads", + &gh__global.prefetch_threads); + if (gh__global.prefetch_threads < 1) + gh__global.prefetch_threads = 1; + + /* + * Read gvfs.postThreads to control parallel POST requests. + * Default to 1 (sequential) for backward compatibility. + */ + gh__global.post_threads = 1; + repo_config_get_int(the_repository, "gvfs.postthreads", + &gh__global.post_threads); + if (gh__global.post_threads < 1) + gh__global.post_threads = 1; + else if (!HAVE_THREADS && gh__global.post_threads > 1) + warning(_("no threads support, ignoring gvfs.postThreads")); + + argc = parse_options(argc, argv, NULL, main_options, main_usage, + PARSE_OPT_STOP_AT_NON_OPTION); + if (argc == 0) + usage_with_options(main_usage, main_options); + + ec = do_sub_cmd(argc, argv); + + gh_http_cleanup(); + + if (ec == GH__ERROR_CODE__USAGE) + usage_with_options(main_usage, main_options); + + return ec; +} diff --git a/gvfs.c b/gvfs.c new file mode 100644 index 00000000000000..5e644f2731bc6e --- /dev/null +++ b/gvfs.c @@ -0,0 +1,61 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "environment.h" +#include "gvfs.h" +#include "setup.h" +#include "config.h" + +static int gvfs_config_loaded; +static struct repository *gvfs_repo; +static int core_gvfs; +static int core_gvfs_is_bool; + +static int early_core_gvfs_config(const char *var, const char *value, + const struct config_context *ctx, void *cb UNUSED) +{ + if (!strcmp(var, "core.gvfs")) + core_gvfs = git_config_bool_or_int("core.gvfs", value, ctx->kvi, + &core_gvfs_is_bool); + if (!strcmp(var, "core.virtualizeobjects")) + core_virtualize_objects = git_config_bool(var, value); + return 0; +} + +static void gvfs_load_config_value(struct repository *r) +{ + if (gvfs_config_loaded && gvfs_repo == r) + return; + + if (r) { + repo_config_get_bool_or_int(r, "core.gvfs", + &core_gvfs_is_bool, &core_gvfs); + repo_config_get_bool(r, "core.virtualizeobjects", + &core_virtualize_objects); + } else if (startup_info->have_repository == 0) { + read_early_config(the_repository, early_core_gvfs_config, NULL); + } else { + repo_config_get_bool_or_int(the_repository, "core.gvfs", + &core_gvfs_is_bool, &core_gvfs); + repo_config_get_bool(the_repository, "core.virtualizeobjects", + &core_virtualize_objects); + } + + /* Turn on all bits if a bool was set in the settings */ + if (core_gvfs_is_bool && core_gvfs) + core_gvfs = -1; + + gvfs_config_loaded = 1; + gvfs_repo = r; +} + +int gvfs_config_is_set(struct repository *r, int mask) +{ + gvfs_load_config_value(r); + return (core_gvfs & mask) == mask; +} + +int gvfs_virtualize_objects(struct repository *r) +{ + gvfs_load_config_value(r); + return core_virtualize_objects; +} diff --git a/gvfs.h b/gvfs.h new file mode 100644 index 00000000000000..f305c5050b8ac0 --- /dev/null +++ b/gvfs.h @@ -0,0 +1,50 @@ +#ifndef GVFS_H +#define GVFS_H + +struct repository; + +/* + * This file is for the specific settings and methods + * used for GVFS functionality + */ + +/* + * The list of bits in the core_gvfs setting + */ +#define GVFS_SKIP_SHA_ON_INDEX (1 << 0) +#define GVFS_BLOCK_COMMANDS (1 << 1) +#define GVFS_MISSING_OK (1 << 2) + +/* + * This behavior of not deleting outside of the sparse-checkout + * is specific to the virtual filesystem support. It is only + * enabled by VFS for Git, and so can be used as an indicator + * that we are in a virtualized filesystem environment and not + * in a Scalar environment. This bit has two names to reflect + * that. + */ +#define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) +#define GVFS_USE_VIRTUAL_FILESYSTEM (1 << 3) + +#define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) +/* Bit 5 was GVFS_LOWER_DEFAULT_SLOP, removed in 2018 (unused). */ +#define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) +#define GVFS_PREFETCH_DURING_FETCH (1 << 7) + +/* + * When set, this flag indicates that the VFS layer supports + * git worktrees. This allows `git worktree add/remove` to + * operate on VFS-enabled repositories. + */ +#define GVFS_SUPPORTS_WORKTREES (1 << 8) + +#define GVFS_ANY_MASK 0xFFFFFFFF + +int gvfs_config_is_set(struct repository *r, int mask); +int gvfs_virtualize_objects(struct repository *r); + +struct object_database; +struct odb_source *add_gvfs_shared_cache_to_alternates(struct object_database *odb, + const char *dir); + +#endif /* GVFS_H */ diff --git a/help.c b/help.c index 46241492cee117..c634a5e521e298 100644 --- a/help.c +++ b/help.c @@ -774,6 +774,22 @@ char *help_unknown_cmd(const char *cmd) exit(1); } +#if defined(__APPLE__) +static const char *git_host_cpu(void) { + if (!strcmp(GIT_HOST_CPU, "universal")) { +#if defined(__x86_64__) + return "x86_64"; +#elif defined(__aarch64__) + return "arm64"; +#endif + } + + return GIT_HOST_CPU; +} +#undef GIT_HOST_CPU +#define GIT_HOST_CPU git_host_cpu() +#endif + void get_version_info(struct strbuf *buf, int show_build_options) { /* diff --git a/hook.c b/hook.c index 5bd0935bae307a..4b1b21839d265c 100644 --- a/hook.c +++ b/hook.c @@ -1,5 +1,9 @@ +#define USE_THE_REPOSITORY_VARIABLE + #include "git-compat-util.h" +#include "trace2/tr2_sid.h" #include "abspath.h" +#include "environment.h" #include "advice.h" #include "config.h" #include "environment.h" @@ -23,16 +27,66 @@ bool is_known_hook(const char *name) return false; } +static int early_hooks_path_config(const char *var, const char *value, + const struct config_context *ctx UNUSED, void *cb) +{ + if (!strcmp(var, "core.hookspath")) + return git_config_pathname((char **)cb, var, value); + + return 0; +} + +/* Discover the hook before setup_git_directory() was called */ +static const char *hook_path_early(const char *name, struct strbuf *result) +{ + static struct strbuf hooks_dir = STRBUF_INIT; + static int initialized; + + if (initialized < 0) + return NULL; + + if (!initialized) { + struct strbuf gitdir = STRBUF_INIT, commondir = STRBUF_INIT; + char *early_hooks_dir = NULL; + + if (discover_git_directory(&commondir, &gitdir) < 0) { + strbuf_release(&gitdir); + strbuf_release(&commondir); + initialized = -1; + return NULL; + } + + read_early_config(the_repository, early_hooks_path_config, &early_hooks_dir); + if (!early_hooks_dir) + strbuf_addf(&hooks_dir, "%s/hooks/", commondir.buf); + else { + strbuf_add_absolute_path(&hooks_dir, early_hooks_dir); + free(early_hooks_dir); + strbuf_addch(&hooks_dir, '/'); + } + + strbuf_release(&gitdir); + strbuf_release(&commondir); + + initialized = 1; + } + + strbuf_addf(result, "%s%s", hooks_dir.buf, name); + return result->buf; +} + const char *find_hook(struct repository *r, const char *name) { static struct strbuf path = STRBUF_INIT; int found_hook; - if (!r || !r->gitdir) - return NULL; - - repo_git_path_replace(r, &path, "hooks/%s", name); + if (!r || !r->gitdir) { + if (!hook_path_early(name, &path)) + return NULL; + } else { + repo_git_path_replace(r, &path, "hooks/%s", name); + } found_hook = access(path.buf, X_OK) >= 0; #ifdef STRIP_EXTENSION if (!found_hook) { @@ -83,14 +137,137 @@ void hook_free(void *p, const char *str UNUSED) free(h); } +static char *get_post_index_change_sentinel_name(struct repository *r) +{ + struct strbuf path = STRBUF_INIT; + const char *sid = tr2_sid_get(); + const char *slash = strchrnul(sid, '/'); + + /* + * Do not write to hooks directory, as it could be redirected + * somewhere like the source tree. + */ + repo_git_path_replace(r, &path, "info/index-change-%.*s.snt", + (int)(slash - sid), sid); + + return strbuf_detach(&path, NULL); +} + +static int write_post_index_change_sentinel(struct repository *r) +{ + char *path = get_post_index_change_sentinel_name(r); + FILE *fp = xfopen(path, "w"); + + if (fp) { + fprintf(fp, "run post-command hook"); + fclose(fp); + } + + free(path); + return fp ? 0 : -1; +} + +/** + * Try to delete the sentinel file for this repository. If that succeeds, then + * return 1. + */ +static int post_index_change_sentinel_exists(struct repository *r) +{ + char *path; + int res = 1; + + /* It can't exist if we don't have a gitdir. */ + if (!r->gitdir) + return 0; + + path = get_post_index_change_sentinel_name(r); + + if (unlink(path)) { + if (is_missing_file_error(errno)) + res = 0; + else + warning_errno("failed to remove index-change sentinel file '%s'", path); + } + + free(path); + return res; +} + +static int check_worktree_change(const char *key, const char *value, + UNUSED const struct config_context *ctx, + void *data) +{ + int *enabled = data; + + if (!strcmp(key, "postcommand.strategy") && + !strcasecmp(value, "worktree-change")) { + *enabled = 1; + return 1; + } + + return 0; +} + +/** + * See if we can replace the requested hook with an internal behavior. + * Returns 0 if the real hook should run. Returns nonzero if we instead + * executed custom internal behavior and the real hook should not run. + */ +static int handle_hook_replacement(struct repository *r, + const char *hook_name, + struct strvec *args) +{ + int enabled = 0; + + read_early_config(r, check_worktree_change, &enabled); + + if (!enabled) + return 0; + + if (!strcmp(hook_name, "post-index-change")) { + /* Create a sentinel file only if the worktree changed. */ + if (!strcmp(args->v[0], "1")) + write_post_index_change_sentinel(r); + + /* We don't skip post-index-change hooks that exist. */ + return 0; + } + if (!strcmp(hook_name, "post-command") && + !post_index_change_sentinel_exists(r)) { + /* We skip the post-command hook in this case. */ + return 1; + } + + return 0; +} + /* Helper to detect and add default "traditional" hooks from the hookdir. */ static void list_hooks_add_default(struct repository *r, const char *hookname, struct string_list *hook_list, struct run_hooks_opt *options) { - const char *hook_path = find_hook(r, hookname); + const char *hook_path; struct hook *h; + /* Interject hook behavior depending on strategy. */ + if (r && options && + handle_hook_replacement(r, hookname, &options->args)) + return; + + hook_path = find_hook(r, hookname); + + /* + * Backwards compatibility hack in VFS for Git: when originally + * introduced (and used!), it was called `post-indexchanged`, but this + * name was changed during the review on the Git mailing list. + * + * Therefore, when the `post-index-change` hook is not found, let's + * look for a hook with the old name (which would be found in case of + * already-existing checkouts). + */ + if (!hook_path && !strcmp(hookname, "post-index-change")) + hook_path = find_hook(r, "post-indexchanged"); + if (!hook_path) return; @@ -543,8 +720,15 @@ struct string_list *list_hooks(struct repository *r, const char *hookname, CALLOC_ARRAY(hook_head, 1); string_list_init_dup(hook_head); - /* Add hooks from the config, e.g. hook.myhook.event = pre-commit */ - list_hooks_add_configured(r, hookname, hook_head, options); + /* + * The pre/post-command hooks are only supported as traditional hookdir + * hooks, never as config-based hooks. Building the config map validates + * all hook.*.event entries and would die() on partially-configured + * hooks, which is fatal when "git config" is still in the middle of + * setting up a multi-key hook definition. + */ + if (strcmp(hookname, "pre-command") && strcmp(hookname, "post-command")) + list_hooks_add_configured(r, hookname, hook_head, options); /* Add the default "traditional" hooks from hookdir. */ list_hooks_add_default(r, hookname, hook_head, options); @@ -850,6 +1034,7 @@ int run_hooks_l(struct repository *r, const char *hook_name, ...) { struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; va_list ap; + int result; const char *arg; va_start(ap, hook_name); @@ -857,5 +1042,7 @@ int run_hooks_l(struct repository *r, const char *hook_name, ...) strvec_push(&opt.args, arg); va_end(ap); - return run_hooks_opt(r, hook_name, &opt); + result = run_hooks_opt(r, hook_name, &opt); + strvec_clear(&opt.args); + return result; } diff --git a/http.c b/http.c index 60c83b778ee568..c3fd4f14278ef9 100644 --- a/http.c +++ b/http.c @@ -1624,6 +1624,64 @@ void http_cleanup(void) FREE_AND_NULL(cached_accept_language); } +static void prepare_curl_handle(CURL *curl) +{ + if (curl_cookie_file && !strcmp(curl_cookie_file, "-")) { + warning(_("refusing to read cookies from http.cookiefile '-'")); + FREE_AND_NULL(curl_cookie_file); + } + curl_easy_setopt(curl, CURLOPT_COOKIEFILE, curl_cookie_file); + if (curl_save_cookies && (!curl_cookie_file || !curl_cookie_file[0])) { + curl_save_cookies = 0; + warning(_("ignoring http.savecookies for empty " + "http.cookiefile")); + } + if (curl_save_cookies) + curl_easy_setopt(curl, CURLOPT_COOKIEJAR, curl_cookie_file); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, pragma_header); + curl_easy_setopt(curl, CURLOPT_RESOLVE, host_resolutions); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, NULL); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, NULL); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, -1L); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 0L); + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(curl, CURLOPT_RANGE, NULL); + + /* + * Default following to off unless "ALWAYS" is configured; this gives + * callers a sane starting point, and they can tweak for individual + * HTTP_FOLLOW_* cases themselves. + */ + if (http_follow_config == HTTP_FOLLOW_ALWAYS) + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + else + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); + + curl_easy_setopt(curl, CURLOPT_IPRESOLVE, git_curl_ipresolve); + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, http_auth_methods); + if (http_auth.password || http_auth.credential || + curl_empty_auth_enabled()) + init_curl_http_auth(curl); +} + +CURL *http_get_curl_handle(void) +{ + CURL *curl = curl_easy_duphandle(curl_default); + + if (!curl) + die("curl_easy_duphandle failed"); + prepare_curl_handle(curl); + return curl; +} + +int http_cookies_configured(void) +{ + return !!curl_cookie_file; +} + struct active_request_slot *get_active_slot(void) { struct active_request_slot *slot = active_queue_head; @@ -1672,44 +1730,8 @@ struct active_request_slot *get_active_slot(void) slot->callback_data = NULL; slot->callback_func = NULL; - if (curl_cookie_file && !strcmp(curl_cookie_file, "-")) { - warning(_("refusing to read cookies from http.cookiefile '-'")); - FREE_AND_NULL(curl_cookie_file); - } - curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file); - if (curl_save_cookies && (!curl_cookie_file || !curl_cookie_file[0])) { - curl_save_cookies = 0; - warning(_("ignoring http.savecookies for empty http.cookiefile")); - } - if (curl_save_cookies) - curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file); - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header); - curl_easy_setopt(slot->curl, CURLOPT_RESOLVE, host_resolutions); + prepare_curl_handle(slot->curl); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL); - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL); - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL); - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, -1L); - curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0L); - curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1L); - curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1L); - curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL); - - /* - * Default following to off unless "ALWAYS" is configured; this gives - * callers a sane starting point, and they can tweak for individual - * HTTP_FOLLOW_* cases themselves. - */ - if (http_follow_config == HTTP_FOLLOW_ALWAYS) - curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L); - else - curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0L); - - curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve); - curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods); - if (http_auth.password || http_auth.credential || curl_empty_auth_enabled()) - init_curl_http_auth(slot->curl); return slot; } diff --git a/http.h b/http.h index 729c51904d39ad..880c7da488cffb 100644 --- a/http.h +++ b/http.h @@ -68,6 +68,8 @@ void step_active_slots(void); void http_init(struct remote *remote, const char *url, int proactive_auth); void http_cleanup(void); +CURL *http_get_curl_handle(void); +int http_cookies_configured(void); struct curl_slist *http_copy_default_headers(void); extern long int git_curl_ipresolve; diff --git a/json-parser.c b/json-parser.c new file mode 100644 index 00000000000000..5d8cd182fb4747 --- /dev/null +++ b/json-parser.c @@ -0,0 +1,191 @@ +#include "git-compat-util.h" +#include "hex.h" +#include "json-parser.h" + +int reset_iterator(struct json_iterator *it) +{ + it->p = it->begin = it->json; + strbuf_release(&it->key); + strbuf_release(&it->string_value); + it->type = JSON_NULL; + return -1; +} + +static int parse_json_string(struct json_iterator *it, struct strbuf *out) +{ + const char *begin = it->p; + + if (*(it->p)++ != '"') { + error("expected double quote: '%.*s'", 5, begin); + return reset_iterator(it); + } + + strbuf_reset(&it->string_value); +#define APPEND(c) strbuf_addch(out, c) + while (*it->p != '"') { + switch (*it->p) { + case '\0': + error("incomplete string: '%s'", begin); + return reset_iterator(it); + case '\\': + it->p++; + if (*it->p == '\\' || *it->p == '"') + APPEND(*it->p); + else if (*it->p == 'b') + APPEND(8); + else if (*it->p == 't') + APPEND(9); + else if (*it->p == 'n') + APPEND(10); + else if (*it->p == 'f') + APPEND(12); + else if (*it->p == 'r') + APPEND(13); + else if (*it->p == 'u') { + unsigned char binary[2]; + int i; + + if (hex_to_bytes(binary, it->p + 1, 2) < 0) { + error("invalid: '%.*s'", 6, it->p - 1); + return reset_iterator(it); + } + it->p += 4; + + i = (binary[0] << 8) | binary[1]; + if (i < 0x80) + APPEND(i); + else if (i < 0x0800) { + APPEND(0xc0 | ((i >> 6) & 0x1f)); + APPEND(0x80 | (i & 0x3f)); + } else if (i < 0x10000) { + APPEND(0xe0 | ((i >> 12) & 0x0f)); + APPEND(0x80 | ((i >> 6) & 0x3f)); + APPEND(0x80 | (i & 0x3f)); + } else { + APPEND(0xf0 | ((i >> 18) & 0x07)); + APPEND(0x80 | ((i >> 12) & 0x3f)); + APPEND(0x80 | ((i >> 6) & 0x3f)); + APPEND(0x80 | (i & 0x3f)); + } + } + break; + default: + APPEND(*it->p); + } + it->p++; + } + + it->end = it->p++; + return 0; +} + +static void skip_whitespace(struct json_iterator *it) +{ + while (isspace(*it->p)) + it->p++; +} + +int iterate_json(struct json_iterator *it) +{ + skip_whitespace(it); + it->begin = it->p; + + switch (*it->p) { + case '\0': + reset_iterator(it); + return 0; + case 'n': + if (!starts_with(it->p, "null")) { + error("unexpected value: %.*s", 4, it->p); + return reset_iterator(it); + } + it->type = JSON_NULL; + it->end = it->p = it->begin + 4; + break; + case 't': + if (!starts_with(it->p, "true")) { + error("unexpected value: %.*s", 4, it->p); + return reset_iterator(it); + } + it->type = JSON_TRUE; + it->end = it->p = it->begin + 4; + break; + case 'f': + if (!starts_with(it->p, "false")) { + error("unexpected value: %.*s", 5, it->p); + return reset_iterator(it); + } + it->type = JSON_FALSE; + it->end = it->p = it->begin + 5; + break; + case '-': case '.': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + it->type = JSON_NUMBER; + it->end = it->p = it->begin + strspn(it->p, "-.0123456789"); + break; + case '"': + it->type = JSON_STRING; + if (parse_json_string(it, &it->string_value) < 0) + return -1; + break; + case '[': { + const char *save = it->begin; + size_t key_offset = it->key.len; + int i = 0, res; + + for (it->p++, skip_whitespace(it); *it->p != ']'; i++) { + strbuf_addf(&it->key, "[%d]", i); + + if ((res = iterate_json(it))) { + reset_iterator(it); + return res; + } + strbuf_setlen(&it->key, key_offset); + + skip_whitespace(it); + if (*it->p == ',') + it->p++; + } + + it->type = JSON_ARRAY; + it->begin = save; + it->end = it->p; + it->p++; + break; + } + case '{': { + const char *save = it->begin; + size_t key_offset = it->key.len; + int res; + + strbuf_addch(&it->key, '.'); + for (it->p++, skip_whitespace(it); *it->p != '}'; ) { + strbuf_setlen(&it->key, key_offset + 1); + if (parse_json_string(it, &it->key) < 0) + return -1; + skip_whitespace(it); + if (*(it->p)++ != ':') { + error("expected colon: %.*s", 5, it->p); + return reset_iterator(it); + } + + if ((res = iterate_json(it))) + return res; + + skip_whitespace(it); + if (*it->p == ',') + it->p++; + } + strbuf_setlen(&it->key, key_offset); + + it->type = JSON_OBJECT; + it->begin = save; + it->end = it->p; + it->p++; + break; + } + } + + return it->fn(it); +} diff --git a/json-parser.h b/json-parser.h new file mode 100644 index 00000000000000..cb1f4832273e57 --- /dev/null +++ b/json-parser.h @@ -0,0 +1,31 @@ +#ifndef JSON_PARSER_H +#define JSON_PARSER_H + +#include "strbuf.h" + +struct json_iterator { + const char *json, *p, *begin, *end; + struct strbuf key, string_value; + enum { + JSON_NULL = 0, + JSON_FALSE, + JSON_TRUE, + JSON_NUMBER, + JSON_STRING, + JSON_ARRAY, + JSON_OBJECT + } type; + int (*fn)(struct json_iterator *it); + void *fn_data; +}; +#define JSON_ITERATOR_INIT(json_, fn_, fn_data_) { \ + .json = json_, .p = json_, \ + .key = STRBUF_INIT, .string_value = STRBUF_INIT, \ + .fn = fn_, .fn_data = fn_data_ \ +} + +int iterate_json(struct json_iterator *it); +/* Releases the iterator, always returns -1 */ +int reset_iterator(struct json_iterator *it); + +#endif diff --git a/mailinfo.c b/mailinfo.c index 13949ff31e1769..36a30c9ce7bff6 100644 --- a/mailinfo.c +++ b/mailinfo.c @@ -1238,11 +1238,11 @@ int mailinfo(struct mailinfo *mi, const char *msg, const char *patch) int mailinfo_parse_quoted_cr_action(const char *actionstr, int *action) { - if (!strcmp(actionstr, "nowarn")) + if (!strcmp(actionstr, "nowarn")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand *action = quoted_cr_nowarn; - else if (!strcmp(actionstr, "warn")) + else if (!strcmp(actionstr, "warn")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand *action = quoted_cr_warn; - else if (!strcmp(actionstr, "strip")) + else if (!strcmp(actionstr, "strip")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand *action = quoted_cr_strip; else return -1; diff --git a/merge-ort.c b/merge-ort.c index c410a5d353234c..04de04a138936c 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -4667,7 +4667,8 @@ static int record_conflicted_index_entries(struct merge_options *opt) */ strmap_for_each_entry(&opt->priv->conflicted, &iter, e) { if (!path_in_sparse_checkout(e->key, index)) { - ensure_full_index(index); + const char *fmt = "merge-ort: path outside sparse checkout (%s)"; + ensure_full_index_with_reason(index, fmt, e->key); break; } } @@ -5457,6 +5458,22 @@ static void merge_recursive_config(struct merge_options *opt, int ui) repo_config_get_int(opt->repo, "merge.verbosity", &opt->verbosity); repo_config_get_int(opt->repo, "diff.renamelimit", &opt->rename_limit); repo_config_get_int(opt->repo, "merge.renamelimit", &opt->rename_limit); + if (!repo_config_get_string(opt->repo, "diff.renamethreshold", &value)) { + const char *arg = value; + opt->rename_score = parse_rename_score(&arg); + if (*arg) + die(_("invalid value for '%s': '%s'"), + "diff.renameThreshold", value); + free(value); + } + if (!repo_config_get_string(opt->repo, "merge.renamethreshold", &value)) { + const char *arg = value; + opt->rename_score = parse_rename_score(&arg); + if (*arg) + die(_("invalid value for '%s': '%s'"), + "merge.renameThreshold", value); + free(value); + } repo_config_get_bool(opt->repo, "merge.renormalize", &renormalize); opt->renormalize = renormalize; if (!repo_config_get_string(opt->repo, "diff.renames", &value)) { diff --git a/meson.build b/meson.build index 79b6af6eaf9641..51a58f61905666 100644 --- a/meson.build +++ b/meson.build @@ -386,6 +386,8 @@ libgit_sources = [ 'gpg-interface.c', 'graph.c', 'grep.c', + 'gvfs.c', + 'gvfs-helper-client.c', 'hash-lookup.c', 'hash.c', 'hashmap.c', @@ -573,6 +575,7 @@ libgit_sources = [ 'utf8.c', 'version.c', 'versioncmp.c', + 'virtualfilesystem.c', 'walker.c', 'wildmatch.c', 'worktree.c', @@ -580,6 +583,8 @@ libgit_sources = [ 'write-or-die.c', 'ws.c', 'wt-status.c', + 'wt-status-deserialize.c', + 'wt-status-serialize.c', 'xdiff-interface.c', 'xdiff/xdiffi.c', 'xdiff/xemit.c', @@ -718,6 +723,7 @@ builtin_sources = [ 'builtin/unpack-file.c', 'builtin/unpack-objects.c', 'builtin/update-index.c', + 'builtin/update-microsoft-git.c', 'builtin/update-ref.c', 'builtin/update-server-info.c', 'builtin/upload-archive.c', @@ -1920,7 +1926,7 @@ test_dependencies += executable('git-http-backend', ) bin_wrappers += executable('scalar', - sources: 'scalar.c', + sources: ['scalar.c', 'json-parser.c'], dependencies: [libgit_commonmain], install: true, install_dir: git_exec_path, @@ -1958,6 +1964,13 @@ if curl.found() ) endif + test_dependencies += executable('git-gvfs-helper', + sources: 'gvfs-helper.c', + dependencies: [libgit_curl], + install: true, + install_dir: get_option('libexecdir') / 'git-core', + ) + foreach alias : [ 'git-remote-https', 'git-remote-ftp', 'git-remote-ftps' ] test_dependencies += executable(alias, sources: 'remote-curl.c', @@ -1992,6 +2005,7 @@ endforeach foreach symlink : [ 'git', + 'git-gvfs-helper', 'git-receive-pack', 'git-shell', 'git-upload-archive', diff --git a/name-hash.c b/name-hash.c index 83757db8746230..9070c1f808365f 100644 --- a/name-hash.c +++ b/name-hash.c @@ -749,6 +749,26 @@ struct cache_entry *index_file_exists(struct index_state *istate, const char *na return NULL; } +struct cache_entry *index_file_next_match(struct index_state *istate, struct cache_entry *ce, int igncase) +{ + struct cache_entry *next; + + if (!igncase || !ce) { + return NULL; + } + + next = hashmap_get_next_entry(&istate->name_hash, ce, ent); + if (!next) + return NULL; + + hashmap_for_each_entry_from(&istate->name_hash, next, ent) { + if (same_name(next, ce->name, ce_namelen(ce), igncase)) + return next; + } + + return NULL; +} + void free_name_hash(struct index_state *istate) { if (!istate->name_hash_initialized) diff --git a/name-hash.h b/name-hash.h index 0cbfc4286316b2..d808eba3e3b672 100644 --- a/name-hash.h +++ b/name-hash.h @@ -12,6 +12,7 @@ int index_dir_find(struct index_state *istate, const char *name, int namelen, void adjust_dirname_case(struct index_state *istate, char *name); struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase); +struct cache_entry *index_file_next_match(struct index_state *istate, struct cache_entry *ce, int igncase); int test_lazy_init_name_hash(struct index_state *istate, int try_threaded); void add_name_hash(struct index_state *istate, struct cache_entry *ce); diff --git a/object-file.h b/object-file.h index ee3aac7965a527..861f4cb75add42 100644 --- a/object-file.h +++ b/object-file.h @@ -24,6 +24,14 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa struct object_info; struct odb_source; +/* + * Add a new object to the loose object cache (possibly after the + * cache was populated). This might be used after dynamically + * fetching a missing object. + */ +void odb_source_loose_cache_add_new_oid(struct odb_source *source, + const struct object_id *oid); + /* * Put in `buf` the name of the file in the local object database that * would be used to store a loose object with the specified oid. diff --git a/object-name.c b/object-name.c index 4eda8c8eac6f9b..f54f210d625422 100644 --- a/object-name.c +++ b/object-name.c @@ -837,7 +837,7 @@ static enum get_oid_result get_parent(struct repository *r, if (ret) return ret; commit = lookup_commit_reference(r, &oid); - if (repo_parse_commit(r, commit)) + if (!commit || repo_parse_commit(r, commit)) return MISSING_OBJECT; if (!idx) { oidcpy(result, &commit->object.oid); diff --git a/odb.c b/odb.c index ac08839bcd361a..95adfdb9f07db8 100644 --- a/odb.c +++ b/odb.c @@ -4,8 +4,11 @@ #include "config.h" #include "environment.h" #include "gettext.h" +#include "gvfs.h" #include "hashmap.h" +#include "gvfs-helper-client.h" #include "hex.h" +#include "hook.h" #include "lockfile.h" #include "loose.h" #include "midx.h" @@ -15,16 +18,21 @@ #include "odb.h" #include "odb/source-inmemory.h" #include "path.h" +#include "pkt-line.h" #include "promisor-remote.h" #include "quote.h" #include "replace-object.h" #include "run-command.h" #include "setup.h" +#include "sigchain.h" #include "strbuf.h" #include "strvec.h" +#include "sub-process.h" #include "submodule.h" #include "tmp-objdir.h" #include "trace2.h" +#include "trace.h" +#include "trace2.h" #include "write-or-die.h" /* @@ -90,22 +98,71 @@ int odb_mkstemp(struct object_database *odb, return xmkstemp_mode(temp_filename->buf, mode); } +static int gvfs_matched_shared_cache_to_alternate; + /* * Return non-zero iff the path is usable as an alternate object database. */ static bool odb_is_source_usable(struct object_database *o, const char *path) { + extern struct strbuf gvfs_shared_cache_pathname; struct strbuf normalized_objdir = STRBUF_INIT; struct hashmap_entry key; bool usable = false; strbuf_realpath(&normalized_objdir, o->sources->path, 1); + if (!strcmp(path, gvfs_shared_cache_pathname.buf)) { + /* + * `gvfs.sharedCache` is the preferred alternate that we + * will use with `gvfs-helper.exe` to dynamically fetch + * missing objects. It is set during git_default_config(). + * + * Make sure the directory exists on disk before we let the + * stock code discredit it. + */ + struct strbuf buf_pack_foo = STRBUF_INIT; + enum scld_error scld; + + /* + * Force create the "" and "/pack" directories, if + * not present on disk. Append an extra bogus directory to + * get safe_create_leading_directories() to see "/pack" + * as a leading directory of something deeper (which it + * won't create). + */ + strbuf_addf(&buf_pack_foo, "%s/pack/foo", path); + + scld = safe_create_leading_directories(o->repo, buf_pack_foo.buf); + if (scld != SCLD_OK && scld != SCLD_EXISTS) { + error_errno(_("could not create shared-cache ODB '%s'"), + gvfs_shared_cache_pathname.buf); + + strbuf_release(&buf_pack_foo); + + /* + * Pretend no shared-cache was requested and + * effectively fallback to ".git/objects" for + * fetching missing objects. + */ + strbuf_release(&gvfs_shared_cache_pathname); + return 0; + } + + /* + * We know that there is an alternate (either from + * .git/objects/info/alternates or from a memory-only + * entry) associated with the shared-cache directory. + */ + gvfs_matched_shared_cache_to_alternate++; + strbuf_release(&buf_pack_foo); + } + /* Detect cases where alternate disappeared */ if (!is_directory(path)) { - error(_("object directory %s does not exist; " - "check .git/objects/info/alternates"), - path); + warning(_("object directory %s does not exist; " + "check .git/objects/info/alternates"), + path); goto out; } @@ -182,8 +239,8 @@ void parse_alternates(const char *string, strbuf_reset(&buf); if (!strbuf_realpath(&buf, pathbuf.buf, 0)) { - error(_("unable to normalize alternate object path: %s"), - pathbuf.buf); + warning(_("unable to normalize alternate object " + "path: %s"), pathbuf.buf); continue; } @@ -238,6 +295,12 @@ static struct odb_source *odb_add_alternate_recursively(struct object_database * return alternate; } +struct odb_source *add_gvfs_shared_cache_to_alternates(struct object_database *odb, + const char *dir) +{ + return odb_add_alternate_recursively(odb, dir, 0); +} + void odb_add_to_alternates_file(struct object_database *odb, const char *dir) { @@ -500,6 +563,7 @@ int odb_for_each_alternate(struct object_database *odb, static void odb_prepare_alternates(struct object_database *odb, const char *alternate_db) { + extern struct strbuf gvfs_shared_cache_pathname; struct strvec sources = STRVEC_INIT; parse_alternates(alternate_db, PATH_SEP, NULL, &sources); @@ -508,6 +572,35 @@ static void odb_prepare_alternates(struct object_database *odb, for (size_t i = 0; i < sources.nr; i++) odb_add_alternate_recursively(odb, sources.v[i], 0); + if (gvfs_shared_cache_pathname.len && + !gvfs_matched_shared_cache_to_alternate) { + /* + * There is no entry in .git/objects/info/alternates for + * the requested shared-cache directory. Therefore, the + * odb-list does not contain this directory. + * + * Force this directory into the odb-list as an in-memory + * alternate. Implicitly create the directory on disk, if + * necessary. + * + * See GIT_ALTERNATE_OBJECT_DIRECTORIES for another example + * of this kind of usage. + * + * Note: This has the net-effect of allowing Git to treat + * `gvfs.sharedCache` as an unofficial alternate. This + * usage should be discouraged for compatbility reasons + * with other tools in the overall Git ecosystem (that + * won't know about this trick). It would be much better + * for us to update .git/objects/info/alternates instead. + * The code here is considered a backstop. + */ + parse_alternates(gvfs_shared_cache_pathname.buf, '\n', NULL, &sources); + odb_source_read_alternates(odb->sources, &sources); + for (size_t i = 0; i < sources.nr; i++) + odb_add_alternate_recursively(odb, sources.v[i], 0); + + } + strvec_clear(&sources); } @@ -516,6 +609,128 @@ int odb_has_alternates(struct object_database *odb) return !!odb->sources->next; } +#define CAP_GET (1u<<0) + +static int subprocess_map_initialized; +static struct hashmap subprocess_map; + +struct read_object_process { + struct subprocess_entry subprocess; + unsigned int supported_capabilities; +}; + +static int start_read_object_fn(struct subprocess_entry *subprocess) +{ + struct read_object_process *entry = (struct read_object_process *)subprocess; + static int versions[] = {1, 0}; + static struct subprocess_capability capabilities[] = { + { "get", CAP_GET }, + { NULL, 0 } + }; + + return subprocess_handshake(subprocess, "git-read-object", versions, + NULL, capabilities, + &entry->supported_capabilities); +} + +int read_object_process(struct repository *r, const struct object_id *oid) +{ + int err; + struct read_object_process *entry; + struct child_process *process; + struct strbuf status = STRBUF_INIT; + const char *cmd = find_hook(r, "read-object"); + uint64_t start; + + if (!cmd) + die(_("could not find the `read-object` hook")); + + start = getnanotime(); + + trace2_region_enter("subprocess", "read_object",r); + + if (!subprocess_map_initialized) { + subprocess_map_initialized = 1; + hashmap_init(&subprocess_map, (hashmap_cmp_fn)cmd2process_cmp, + NULL, 0); + entry = NULL; + } else { + entry = (struct read_object_process *) subprocess_find_entry(&subprocess_map, cmd); + } + + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->supported_capabilities = 0; + + if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, + start_read_object_fn)) { + free(entry); + err = -1; + goto leave_region; + } + } + process = &entry->subprocess.process; + + if (!(CAP_GET & entry->supported_capabilities)) { + err = -1; + goto leave_region; + } + + sigchain_push(SIGPIPE, SIG_IGN); + + err = packet_write_fmt_gently(process->in, "command=get\n"); + if (err) + goto done; + + err = packet_write_fmt_gently(process->in, "sha1=%s\n", oid_to_hex(oid)); + if (err) + goto done; + + err = packet_flush_gently(process->in); + if (err) + goto done; + + err = subprocess_read_status(process->out, &status); + err = err ? err : strcmp(status.buf, "success"); + +done: + sigchain_pop(SIGPIPE); + + if (err || errno == EPIPE) { + err = err ? err : errno; + if (!strcmp(status.buf, "error")) { + /* The process signaled a problem with the file. */ + } + else if (!strcmp(status.buf, "abort")) { + /* + * The process signaled a permanent problem. Don't try to read + * objects with the same command for the lifetime of the current + * Git process. + */ + entry->supported_capabilities &= ~CAP_GET; + } + else { + /* + * Something went wrong with the read-object process. + * Force shutdown and restart if needed. + */ + error("external process '%s' failed", cmd); + subprocess_stop(&subprocess_map, + (struct subprocess_entry *)entry); + free(entry); + } + } + + trace_performance_since(start, "read_object_process"); + +leave_region: + trace2_region_leave_printf("subprocess", "read_object", r, + "result %d", err); + + strbuf_release(&status); + return err; +} + int obj_read_use_lock = 0; pthread_mutex_t obj_read_mutex; @@ -537,6 +752,27 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } +static enum odb_read_status read_object_info_from_sources( + struct object_database *odb, const struct object_id *oid, + struct object_info *oi, enum object_info_flags flags, + struct strbuf *errmsg) +{ + struct odb_source *source; + enum odb_read_status ret = ODB_READ_NOT_FOUND; + + for (source = odb->sources; source; source = source->next) { + enum odb_read_status source_ret = odb_source_read_object_info( + source, oid, oi, flags, errmsg->len ? NULL : errmsg); + + if (!source_ret) + return ODB_READ_OK; + if (source_ret != ODB_READ_NOT_FOUND) + ret = source_ret; + } + + return ret; +} + static enum odb_read_status do_oid_object_info_extended(struct object_database *odb, const struct object_id *oid, struct object_info *oi, unsigned flags) @@ -546,6 +782,8 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * enum odb_read_status ret; int already_retried = 0; bool corrupt = false; + int tried_hook = 0; + int tried_gvfs_helper = 0; if (flags & OBJECT_INFO_LOOKUP_REPLACE) real = lookup_replace_object(odb->repo, oid); @@ -553,19 +791,51 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * if (is_null_oid(real)) return -1; +retry: if (!odb_source_read_object_info(odb->inmemory_objects, oid, oi, flags, NULL)) return 0; while (1) { - struct odb_source *source; + extern int core_use_gvfs_helper; + enum object_info_flags source_flags = flags; - for (source = odb->sources; source; source = source->next) { - ret = odb_source_read_object_info(source, real, oi, flags, - corrupt_err.len ? NULL : &corrupt_err); + if (odb_has_alternates(odb)) { + ret = read_object_info_from_sources(odb, real, oi, + flags | OBJECT_INFO_SKIP_LOOSE, &corrupt_err); if (!ret) goto out; if (ret != ODB_READ_NOT_FOUND) corrupt = true; + source_flags |= OBJECT_INFO_SKIP_PACKED; + } + + ret = read_object_info_from_sources(odb, real, oi, source_flags, + &corrupt_err); + if (!ret) + goto out; + if (ret != ODB_READ_NOT_FOUND) + corrupt = true; + + if (core_use_gvfs_helper && !tried_gvfs_helper && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + enum gh_client__created ghc; + + gh_client__get_immediate(real, &ghc); + tried_gvfs_helper = 1; + + /* + * Retry the lookup IIF `gvfs-helper` created one + * or more new packfiles or loose objects. + */ + if (ghc != GHC__CREATED__NOTHING) + continue; + + /* + * If `gvfs-helper` fails, we just want to return -1. + * But allow the other providers to have a shot at it. + * (At least until we have a chance to consolidate + * them.) + */ } /* @@ -574,14 +844,22 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * * caches or reload on-disk state. */ if (!(flags & OBJECT_INFO_QUICK)) { - for (source = odb->sources; source; source = source->next) { - ret = odb_source_read_object_info(source, real, oi, - flags | OBJECT_INFO_SECOND_READ, - corrupt_err.len ? NULL : &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; + ret = read_object_info_from_sources(odb, real, oi, + flags | OBJECT_INFO_SECOND_READ, + &corrupt_err); + if (!ret) + goto out; + if (ret != ODB_READ_NOT_FOUND) + corrupt = true; + if (gvfs_virtualize_objects(odb->repo) && !tried_hook && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + // TODO Assert or at least trace2 if gvfs-helper + // TODO was tried and failed and then read-object-hook + // TODO is successful at getting this object. + tried_hook = 1; + // TODO BUG? Should 'oid' be 'real' ? + if (!read_object_process(odb->repo, oid)) + goto retry; } } @@ -748,6 +1026,9 @@ void *odb_read_object(struct object_database *odb, unsigned flags = OBJECT_INFO_DIE_IF_CORRUPT | OBJECT_INFO_LOOKUP_REPLACE; void *data; + if (gvfs_config_is_set(odb->repo, GVFS_MISSING_OK)) + flags &= ~OBJECT_INFO_DIE_IF_CORRUPT; + oi.typep = type; oi.sizep = size; oi.contentp = &data; @@ -822,11 +1103,13 @@ int odb_has_object(struct object_database *odb, const struct object_id *oid, } int odb_freshen_object(struct object_database *odb, - const struct object_id *oid) + const struct object_id *oid, + int skip_virtualized_objects) { struct odb_source *source; for (source = odb->sources; source; source = source->next) - if (odb_source_freshen_object(source, oid, NULL)) + if (odb_source_freshen_object(source, oid, NULL, + skip_virtualized_objects)) return 1; return 0; } @@ -995,7 +1278,7 @@ int odb_write_object_ext(struct object_database *odb, * We can skip the write in case we already have the object available. * In that case, we only freshen its mtime. */ - if (odb_freshen_object(odb, oid)) + if (odb_freshen_object(odb, oid, 1)) return 0; if (compat) { diff --git a/odb.h b/odb.h index afe04d5ff8be04..f6deef02f6b2cf 100644 --- a/odb.h +++ b/odb.h @@ -445,6 +445,18 @@ enum object_info_flags { */ OBJECT_INFO_SECOND_READ = (1 << 4), + /* + * Only consult the packed object store of a source, skipping its loose + * object store (OBJECT_INFO_SKIP_LOOSE), or vice versa + * (OBJECT_INFO_SKIP_PACKED). These are used by + * odb_read_object_info_extended() to scan the packfiles of all sources + * before consulting any source's loose object store, so that an object + * that resides in an alternate's packfile is not preceded by a spurious + * loose-object lookup on an earlier source. + */ + OBJECT_INFO_SKIP_LOOSE = (1 << 5), + OBJECT_INFO_SKIP_PACKED = (1 << 6), + /* * This is meant for bulk prefetching of missing blobs in a partial * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK. @@ -496,7 +508,8 @@ int odb_has_object(struct object_database *odb, enum odb_has_object_flags flags); int odb_freshen_object(struct object_database *odb, - const struct object_id *oid); + const struct object_id *oid, + int skip_virtualized_objects); void odb_assert_oid_type(struct object_database *odb, const struct object_id *oid, enum object_type expect); @@ -796,6 +809,9 @@ struct odb_generate_pack_options { /* Do not use bitmap indices when computing reachability. */ unsigned disable_bitmaps:1; + + /* Do not reuse deltas. */ + unsigned no_reuse_delta:1; }; #define ODB_GENERATE_PACK_OPTIONS_INIT { \ @@ -859,4 +875,6 @@ void parse_alternates(const char *string, const char *relative_base, struct strvec *out); +int read_object_process(struct repository *r, const struct object_id *oid); + #endif /* ODB_H */ diff --git a/odb/source-files.c b/odb/source-files.c index 109c416957fd53..dc212ccb1e0d76 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -83,17 +83,23 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source struct strbuf *errmsg) { struct odb_source_files *files = odb_source_files_downcast(source); - enum odb_read_status ret_packed, ret_loose; - - ret_packed = odb_source_read_object_info(&files->packed->base, oid, oi, - flags, errmsg); - if (!ret_packed) - return 0; + enum odb_read_status ret_packed = ODB_READ_NOT_FOUND; + enum odb_read_status ret_loose = ODB_READ_NOT_FOUND; + + if (!(flags & OBJECT_INFO_SKIP_PACKED)) { + ret_packed = odb_source_read_object_info(&files->packed->base, + oid, oi, flags, errmsg); + if (!ret_packed) + return 0; + } - ret_loose = odb_source_read_object_info(&files->loose->base, oid, oi, flags, - ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL); - if (!ret_loose) - return 0; + if (!(flags & OBJECT_INFO_SKIP_LOOSE)) { + ret_loose = odb_source_read_object_info(&files->loose->base, + oid, oi, flags, + ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL); + if (!ret_loose) + return 0; + } /* * Reading the packed object may have failed even though the object @@ -195,11 +201,14 @@ static int odb_source_files_find_abbrev_len(struct odb_source *source, static int odb_source_files_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects) { struct odb_source_files *files = odb_source_files_downcast(source); - if (odb_source_freshen_object(&files->packed->base, oid, mtime) || - odb_source_freshen_object(&files->loose->base, oid, mtime)) + if (odb_source_freshen_object(&files->packed->base, oid, mtime, + skip_virtualized_objects) || + odb_source_freshen_object(&files->loose->base, oid, mtime, + skip_virtualized_objects)) return 1; return 0; } @@ -843,6 +852,8 @@ static int odb_source_files_generate_pack(struct odb_source *source UNUSED, strvec_push(&cp->args, "--missing=allow-promisor"); if (opts->disable_bitmaps) strvec_push(&cp->args, "--no-use-bitmap-index"); + if (opts->no_reuse_delta) + strvec_push(&cp->args, "--no-reuse-delta"); switch (opts->progress) { case ODB_GENERATE_PACK_PROGRESS_NONE: strvec_push(&cp->args, "--quiet"); diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 473085fa0e3785..ed5db345c957ae 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -307,7 +307,8 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, static int odb_source_inmemory_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime UNUSED) + const time_t *mtime UNUSED, + int skip_virtualized_objects UNUSED) { struct odb_source_inmemory *inmemory = odb_source_inmemory_downcast(source); if (find_cached_object(inmemory, oid)) diff --git a/odb/source-loose.c b/odb/source-loose.c index e76efd9e37d54f..94d2372669aea0 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -2,6 +2,7 @@ #include "abspath.h" #include "chdir-notify.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "loose.h" #include "object-file.h" @@ -587,12 +588,22 @@ static int odb_source_loose_count_objects(struct odb_source *source, static int odb_source_loose_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects) { struct odb_source_loose *loose = odb_source_loose_downcast(source); static struct strbuf path = STRBUF_INIT; + int ret, tried_hook = 0; odb_loose_path(loose, &path, oid); - return !!check_and_freshen_file(path.buf, 1, mtime); +retry: + ret = !!check_and_freshen_file(path.buf, 1, mtime); + if (!ret && gvfs_virtualize_objects(source->odb->repo) && + !skip_virtualized_objects && !tried_hook) { + tried_hook = 1; + if (!read_object_process(source->odb->repo, oid)) + goto retry; + } + return ret; } /* Finalize a file on disk, and close it. */ @@ -946,7 +957,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, die(_("deflateEnd on stream object failed (%d)"), ret); close_loose_object(loose, fd, tmp_file.buf); - if (odb_freshen_object(loose->base.odb, oid)) { + if (odb_freshen_object(loose->base.odb, oid, 1)) { unlink_or_warn(tmp_file.buf); goto cleanup; } @@ -997,6 +1008,27 @@ static int odb_source_loose_write_alternate(struct odb_source *source UNUSED, return error("loose source does not support alternates"); } +void odb_source_loose_cache_add_new_oid(struct odb_source *source, + const struct object_id *oid) +{ + struct odb_source_loose *loose; + struct oidtree *cache; + + switch (source->type) { + case ODB_SOURCE_FILES: + loose = odb_source_files_downcast(source)->loose; + break; + case ODB_SOURCE_LOOSE: + loose = odb_source_loose_downcast(source); + break; + default: + BUG("source of type '%d' has no loose cache", source->type); + } + + cache = odb_source_loose_cache(loose, oid); + append_loose_object(oid, NULL, cache); +} + static void odb_source_loose_clear_cache(struct odb_source_loose *loose) { oidtree_clear(loose->cache); diff --git a/odb/source-packed.c b/odb/source-packed.c index a26c323dfa9716..ef09fde1b91c9b 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -636,7 +636,8 @@ static int odb_source_packed_find_abbrev_len(struct odb_source *source, static int odb_source_packed_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects UNUSED) { struct odb_source_packed *packed = odb_source_packed_downcast(source); struct utimbuf times, *timesp = NULL; diff --git a/odb/source.h b/odb/source.h index 13c7cd61a60013..cafeef1f8f8367 100644 --- a/odb/source.h +++ b/odb/source.h @@ -136,6 +136,10 @@ struct odb_source { * second read in case they know that the first read would have * already surfaced the object without reloading any on-disk state. * + * - `OBJECT_INFO_SKIP_LOOSE` and `OBJECT_INFO_SKIP_PACKED` tell the + * files backend not to consult its loose or packed source, + * respectively. + * * The callback is expected to return an `enum odb_read_status`. Please * refer to the individual values that can be returned. In case reading * the object has failed with a generic error and `errmsg` is non-NULL, @@ -221,7 +225,8 @@ struct odb_source { */ int (*freshen_object)(struct odb_source *source, const struct object_id *oid, - const time_t *mtime); + const time_t *mtime, + int skip_virtualized_objects); /* * This callback is expected to persist the given object into the @@ -495,9 +500,11 @@ static inline int odb_source_find_abbrev_len(struct odb_source *source, */ static inline int odb_source_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects) { - return source->freshen_object(source, oid, mtime); + return source->freshen_object(source, oid, mtime, + skip_virtualized_objects); } /* diff --git a/odb/streaming.c b/odb/streaming.c index 8f2143cab5d846..282c2412f1e0a2 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -6,6 +6,7 @@ #include "convert.h" #include "environment.h" #include "repository.h" +#include "gvfs.h" #include "odb.h" #include "odb/source.h" #include "odb/streaming.h" @@ -157,13 +158,14 @@ static int open_istream_incore(struct odb_stream **out, .base.read = read_istream_incore, }; struct odb_incore_read_stream *st; + unsigned flags = gvfs_config_is_set(odb->repo, GVFS_MISSING_OK) ? + 0 : OBJECT_INFO_DIE_IF_CORRUPT; int ret; oi.typep = &stream.base.type; oi.sizep = &stream.base.size; oi.contentp = (void **)&stream.buf; - ret = odb_read_object_info_extended(odb, oid, &oi, - OBJECT_INFO_DIE_IF_CORRUPT); + ret = odb_read_object_info_extended(odb, oid, &oi, flags); if (ret) return ret; diff --git a/pack-mtimes.c b/pack-mtimes.c index 8e1f2dec0ef62f..8dfe5017dfd267 100644 --- a/pack-mtimes.c +++ b/pack-mtimes.c @@ -28,7 +28,7 @@ static int load_pack_mtimes_file(char *mtimes_file, int fd, ret = 0; struct stat st; uint32_t *data = NULL; - size_t mtimes_size, expected_size; + size_t mtimes_size = 0, expected_size; struct mtimes_header header; fd = git_open(mtimes_file); diff --git a/pack-revindex.c b/pack-revindex.c index 62387ae6320181..282563f5f98729 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -211,7 +211,7 @@ static int load_revindex_from_disk(const struct git_hash_algo *algo, int fd, ret = 0; struct stat st; void *data = NULL; - size_t revindex_size; + size_t revindex_size = 0; struct revindex_header *hdr; if (git_env_bool(GIT_TEST_REV_INDEX_DIE_ON_DISK, 0)) diff --git a/packfile.c b/packfile.c index 4fa5fd67c8497f..702cf33622054a 100644 --- a/packfile.c +++ b/packfile.c @@ -1501,6 +1501,13 @@ struct unpack_entry_stack_ent { size_t size; }; +static unsigned long g_nr_unpack_entry; + +unsigned long get_nr_unpack_entry(void) +{ + return g_nr_unpack_entry; +} + void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, enum object_type *final_type, size_t *final_size) { @@ -1514,6 +1521,8 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC; int base_from_cache = 0; + g_nr_unpack_entry++; + prepare_repo_settings(p->repo); write_pack_access_log(p, obj_offset); diff --git a/packfile.h b/packfile.h index 6d30d15a0053b3..b3c66273f6272c 100644 --- a/packfile.h +++ b/packfile.h @@ -359,4 +359,9 @@ int load_idx(const char *path, const unsigned int hashsz, void *idx_map, */ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *len); +/* + * Return the number of objects fetched from a packfile. + */ +unsigned long get_nr_unpack_entry(void); + #endif diff --git a/pkt-line.c b/pkt-line.c index 3fc3e9ea7059be..e479a4c3e13157 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -230,7 +230,7 @@ static int do_packet_write(const int fd_out, const char *buf, size_t size, return 0; } -static int packet_write_gently(const int fd_out, const char *buf, size_t size) +int packet_write_gently(const int fd_out, const char *buf, size_t size) { struct strbuf err = STRBUF_INIT; if (do_packet_write(fd_out, buf, size, &err)) { diff --git a/pkt-line.h b/pkt-line.h index e6cf85e34ee3c4..5c43e253993983 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -29,6 +29,7 @@ void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +int packet_write_gently(const int fd_out, const char *buf, size_t size); int write_packetized_from_fd_no_flush(int fd_in, int fd_out); int write_packetized_from_buf_no_flush_count(const char *src_in, size_t len, int fd_out, int *packet_counter); diff --git a/promisor-remote.c b/promisor-remote.c index 43505d1e1ac8fd..f85c7e7085cd8d 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -1,7 +1,9 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "environment.h" #include "gettext.h" +#include "gvfs-helper-client.h" #include "hex.h" #include "odb.h" #include "promisor-remote.h" @@ -223,7 +225,7 @@ struct promisor_remote *repo_promisor_remote_find(struct repository *r, int repo_has_promisor_remote(struct repository *r) { - return !!repo_promisor_remote_find(r, NULL); + return core_use_gvfs_helper || !!repo_promisor_remote_find(r, NULL); } int repo_has_accepted_promisor_remote(struct repository *r) @@ -306,6 +308,15 @@ void promisor_remote_get_direct(struct repository *repo, if (oid_nr == 0) return; + if (core_use_gvfs_helper) { + enum gh_client__created ghc = GHC__CREATED__NOTHING; + + trace2_data_intmax("bug", the_repository, "fetch_objects/gvfs-helper", oid_nr); + gh_client__queue_oid_array(oids, oid_nr); + if (!gh_client__drain_queue(&ghc)) + return; + die(_("failed to fetch missing objects from the remote")); + } promisor_remote_init(repo); diff --git a/prompt.c b/prompt.c index d8d74c7e379dbe..e15f9e344f6972 100644 --- a/prompt.c +++ b/prompt.c @@ -38,7 +38,7 @@ static char *do_askpass(const char *cmd, const char *prompt) return NULL; } - strbuf_setlen(&buffer, strcspn(buffer.buf, "\r\n")); + strbuf_setlen(&buffer, strcspn(buffer.buf, "\r\n")); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand return buffer.buf; } diff --git a/read-cache-ll.h b/read-cache-ll.h index d279059b5fa428..31fa370e43d379 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -118,7 +118,7 @@ static inline unsigned create_ce_flags(unsigned stage) #define ce_namelen(ce) ((ce)->ce_namelen) #define ce_size(ce) cache_entry_size(ce_namelen(ce)) #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT) -#define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) +#define ce_uptodate(ce) (((ce)->ce_flags & CE_UPTODATE) || ((ce)->ce_flags & CE_FSMONITOR_VALID)) #define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE) #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) #define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD) diff --git a/read-cache.c b/read-cache.c index 4cf60d6776e5a2..7653d13fa36e6f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -8,6 +8,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "virtualfilesystem.h" #include "config.h" #include "date.h" #include "diff.h" @@ -555,7 +556,9 @@ static int index_name_stage_pos(struct index_state *istate, if (S_ISSPARSEDIR(ce->ce_mode) && ce_namelen(ce) < namelen && !strncmp(name, ce->name, ce_namelen(ce))) { - ensure_full_index(istate); + const char *fmt = "searching for '%s' and found parent dir '%s'"; + ensure_full_index_with_reason(istate, fmt, + name, ce->name); return index_name_stage_pos(istate, name, namelen, stage, search_mode); } } @@ -1750,7 +1753,10 @@ static int read_index_extension(struct index_state *istate, { switch (CACHE_EXT(ext)) { case CACHE_EXT_TREE: + trace2_region_enter("index", "read/extension/cache_tree", NULL); istate->cache_tree = cache_tree_read(istate->repo, data, sz); + trace2_data_intmax("index", NULL, "read/extension/cache_tree/bytes", (intmax_t)sz); + trace2_region_leave("index", "read/extension/cache_tree", NULL); break; case CACHE_EXT_RESOLVE_UNDO: istate->resolve_undo = resolve_undo_read(data, sz, the_hash_algo); @@ -1967,6 +1973,7 @@ static void post_read_index_from(struct index_state *istate) tweak_untracked_cache(istate); tweak_split_index(istate); tweak_fsmonitor(istate); + apply_virtualfilesystem(istate); } static size_t estimate_cache_size_from_compressed(unsigned int entries) @@ -2039,6 +2046,17 @@ static void *load_index_extensions(void *_data) return NULL; } +static void *load_index_extensions_threadproc(void *_data) +{ + void *result; + + trace2_thread_start("load_index_extensions"); + result = load_index_extensions(_data); + trace2_thread_exit(); + + return result; +} + /* * A helper function that will load the specified range of cache entries * from the memory mapped file and add them to the given index. @@ -2115,12 +2133,17 @@ static void *load_cache_entries_thread(void *_data) struct load_cache_entries_thread_data *p = _data; int i; + trace2_thread_start("load_cache_entries"); + /* iterate across all ieot blocks assigned to this thread */ for (i = p->ieot_start; i < p->ieot_start + p->ieot_blocks; i++) { p->consumed += load_cache_entry_block(p->istate, p->ce_mem_pool, p->offset, p->ieot->entries[i].nr, p->mmap, p->ieot->entries[i].offset, NULL); p->offset += p->ieot->entries[i].nr; } + + trace2_thread_exit(); + return NULL; } @@ -2290,7 +2313,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) int err; p.src_offset = extension_offset; - err = pthread_create(&p.pthread, NULL, load_index_extensions, &p); + err = pthread_create(&p.pthread, NULL, load_index_extensions_threadproc, &p); if (err) die(_("unable to create load_index_extensions thread: %s"), strerror(err)); @@ -2338,7 +2361,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) */ prepare_repo_settings(istate->repo); if (istate->repo->settings.command_requires_full_index) - ensure_full_index(istate); + ensure_full_index_with_reason(istate, "incompatible builtin"); else ensure_correct_sparsity(istate); @@ -2546,7 +2569,7 @@ int repo_index_has_changes(struct repository *repo, return opt.flags.has_changes != 0; } else { /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(istate); + ensure_full_index_unaudited(istate); for (i = 0; sb && i < istate->cache_nr; i++) { if (i) strbuf_addch(sb, ' '); @@ -3014,9 +3037,13 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, !drop_cache_tree && istate->cache_tree) { strbuf_reset(&sb); + trace2_region_enter("index", "write/extension/cache_tree", NULL); cache_tree_write(istate->repo, &sb, istate->cache_tree); err = write_index_ext_header(f, eoie_c, CACHE_EXT_TREE, sb.len) < 0; hashwrite(f, sb.buf, sb.len); + trace2_data_intmax("index", NULL, "write/extension/cache_tree/bytes", (intmax_t)sb.len); + trace2_region_leave("index", "write/extension/cache_tree", NULL); + if (err) { ret = -1; goto out; @@ -3156,7 +3183,7 @@ static int do_write_locked_index(struct index_state *istate, "%s", get_lock_file_path(lock)); if (was_full) - ensure_full_index(istate); + ensure_full_index_with_reason(istate, "re-expanding after write"); if (ret) return ret; @@ -3271,7 +3298,7 @@ static int write_shared_index(struct index_state *istate, the_repository, "%s", get_tempfile_path(*temp)); if (was_full) - ensure_full_index(istate); + ensure_full_index_with_reason(istate, "re-expanding after write"); if (ret) return ret; @@ -3831,7 +3858,7 @@ void overlay_tree_on_index(struct index_state *istate, /* Hoist the unmerged entries up to stage #3 to make room */ /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(istate); + ensure_full_index_unaudited(istate); for (i = 0; i < istate->cache_nr; i++) { struct cache_entry *ce = istate->cache[i]; if (!ce_stage(ce)) @@ -3997,7 +4024,7 @@ static void update_callback(struct diff_queue_struct *q, struct diff_filepair *p = q->queue[i]; const char *path = p->one->path; - if (!data->include_sparse && + if (!data->include_sparse && !core_virtualfilesystem && !path_in_sparse_checkout(path, data->index)) continue; diff --git a/refs.c b/refs.c index 92d5df5b71fa4b..f598ea6e4eeed5 100644 --- a/refs.c +++ b/refs.c @@ -397,7 +397,7 @@ int refname_is_safe(const char *refname) * For example: refs/foo/../bar is safe but refs/foo/../../bar * is not. */ - buf = xmallocz(restlen); + buf = xmallocz(restlen); // CodeQL [SM01952] justification: CodeQL fails to recognize that xmallocz() accounts for the NUL terminator, instead assuming malloc() semantics result = !normalize_path_copy(buf, rest) && !strcmp(buf, rest); free(buf); return result; diff --git a/remote-curl.c b/remote-curl.c index 2c35dd52400f83..e657c6c78b7c3a 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -212,6 +212,7 @@ static int set_option(const char *name, size_t namelen, const char *value) options.refetch = 1; return 0; } else if (!strncmp(name, "filter", namelen)) { + free(options.filter); options.filter = xstrdup(value); return 0; } else if (!strncmp(name, "object-format", namelen)) { @@ -1200,6 +1201,9 @@ static int fetch_git(struct discovery *heads, struct strvec args = STRVEC_INIT; struct strbuf rpc_result = STRBUF_INIT; + if (core_use_gvfs_helper) + return 0; + strvec_pushl(&args, "fetch-pack", "--stateless-rpc", "--stdin", "--lock-pack", NULL); if (options.followtags) diff --git a/remote.c b/remote.c index fe620684635620..1f79407caa9a88 100644 --- a/remote.c +++ b/remote.c @@ -22,6 +22,7 @@ #include "setup.h" #include "string-list.h" #include "strvec.h" +#include "trace2.h" #include "commit-reach.h" #include "advice.h" #include "connect.h" @@ -2498,8 +2499,16 @@ int format_tracking_info(struct branch *branch, struct strbuf *sb, if (is_upstream && (!push_ref || !strcmp(upstream_ref, push_ref))) is_push = 1; + trace2_region_enter("tracking", "stat_tracking_pair", NULL); cmp = stat_branch_pair(branch->refname, full_ref, &ours, &theirs, abf); + trace2_data_intmax("tracking", NULL, "stat_tracking_pair/ab_flags", abf); + trace2_data_intmax("tracking", NULL, "stat_tracking_pair/ab_result", cmp); + if (cmp >= 0 && abf == AHEAD_BEHIND_FULL) { + trace2_data_intmax("tracking", NULL, "stat_tracking_pair/ab_ahead", ours); + trace2_data_intmax("tracking", NULL, "stat_tracking_pair/ab_behind", theirs); + } + trace2_region_leave("tracking", "stat_tracking_pair", NULL); if (cmp < 0) { if (is_upstream) { diff --git a/repo-settings.c b/repo-settings.c index f3be3b8c5a3d09..7ea244a140e423 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -5,6 +5,7 @@ #include "midx.h" #include "pack-objects.h" #include "setup.h" +#include "gvfs.h" static void repo_cfg_bool(struct repository *r, const char *key, int *dest, int def) @@ -77,7 +78,7 @@ void prepare_repo_settings(struct repository *r) repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1); repo_cfg_bool(r, "pack.usepathwalk", &r->settings.pack_use_path_walk, 0); repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); - repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); + repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 1); repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash); repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1); repo_cfg_bool(r, "pack.usebitmapboundarytraversal", @@ -85,6 +86,13 @@ void prepare_repo_settings(struct repository *r) r->settings.pack_use_bitmap_boundary_traversal); repo_cfg_bool(r, "core.usereplacerefs", &r->settings.read_replace_refs, 1); + /* + * For historical compatibility reasons, enable index.skipHash based + * on a bit in core.gvfs. + */ + if (gvfs_config_is_set(r, GVFS_SKIP_SHA_ON_INDEX)) + r->settings.index_skip_hash = 1; + /* * The GIT_TEST_MULTI_PACK_INDEX variable is special in that * either it *or* the config sets diff --git a/repository.c b/repository.c index 74e7d9527ea4c3..bb88f6b5fc6daf 100644 --- a/repository.c +++ b/repository.c @@ -457,7 +457,7 @@ int repo_read_index(struct repository *repo) prepare_repo_settings(repo); if (repo->settings.command_requires_full_index) - ensure_full_index(repo->index); + ensure_full_index_with_reason(repo->index, "incompatible builtin"); /* * If sparse checkouts are in use, check whether paths with the diff --git a/resolve-undo.c b/resolve-undo.c index 52c45e5a494636..b7eb904a8179d7 100644 --- a/resolve-undo.c +++ b/resolve-undo.c @@ -161,7 +161,7 @@ void unmerge_index(struct index_state *istate, const struct pathspec *pathspec, return; /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(istate); + ensure_full_index_unaudited(istate); for_each_string_list_item(item, istate->resolve_undo) { const char *path = item->string; diff --git a/revision.c b/revision.c index ee1df92d1d0779..976f2d5888d79d 100644 --- a/revision.c +++ b/revision.c @@ -1808,7 +1808,7 @@ static void do_add_index_objects_to_pending(struct rev_info *revs, int i; /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(istate); + ensure_full_index_unaudited(istate); for (i = 0; i < istate->cache_nr; i++) { struct cache_entry *ce = istate->cache[i]; struct blob *blob; @@ -3466,6 +3466,9 @@ static int leave_one_treesame_to_parent(struct rev_info *revs, struct commit *co struct commit_list *p; unsigned n; + if (!ts) + return 0; + for (p = commit->parents, n = 0; p; p = p->next, n++) { if (ts->treesame[n]) { if (p->item->object.flags & TMP_MARK) { diff --git a/scalar.c b/scalar.c index a80d8ee3ff54a7..00534eb4847e16 100644 --- a/scalar.c +++ b/scalar.c @@ -7,19 +7,42 @@ #include "git-compat-util.h" #include "abspath.h" #include "gettext.h" +#include "hex.h" #include "parse-options.h" #include "config.h" +#include "environment.h" #include "run-command.h" #include "simple-ipc.h" #include "fsmonitor-ipc.h" #include "fsmonitor-settings.h" #include "refs.h" #include "dir.h" +#include "object-file.h" #include "packfile.h" #include "help.h" #include "setup.h" +#include "wrapper.h" #include "trace2.h" #include "path.h" +#include "json-parser.h" +#include "gvfs.h" +#include "gvfs-helper-client.h" +#include "object-name.h" +#include "remote.h" +#include "path.h" + +/* + * The `core.gvfs` bitmask that `scalar clone` configures for enlistments + * that use the GVFS Protocol (historically the value `150`). See gvfs.h + * for the meaning of the individual bits. + */ +#define SCALAR_GVFS_MODE (GVFS_BLOCK_COMMANDS | GVFS_MISSING_OK | \ + GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK | \ + GVFS_PREFETCH_DURING_FETCH) + +static int is_unattended(void) { + return git_env_bool("Scalar_UNATTENDED", 0); +} static void setup_enlistment_directory(int argc, const char **argv, const char * const *usagestr, @@ -47,6 +70,9 @@ static void setup_enlistment_directory(int argc, const char **argv, die(_("need a working directory")); strbuf_trim_trailing_dir_sep(&path); +#ifdef GIT_WINDOWS_NATIVE + convert_slashes(path.buf); +#endif /* check if currently in enlistment root with src/ workdir */ len = path.len; @@ -73,21 +99,56 @@ static void setup_enlistment_directory(int argc, const char **argv, strbuf_release(&path); } +static int git_retries = 3; + +static int run_git_argv(const struct strvec *argv) +{ + int res = 0, attempts; + + for (attempts = 0, res = 1; + res && attempts < git_retries; + attempts++) { + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushv(&cmd.args, argv->v); + res = run_command(&cmd); + } + + return res; +} + LAST_ARG_MUST_BE_NULL static int run_git(const char *arg, ...) { - struct child_process cmd = CHILD_PROCESS_INIT; va_list args; const char *p; + struct strvec argv = STRVEC_INIT; + int res; va_start(args, arg); - strvec_push(&cmd.args, arg); + strvec_push(&argv, arg); while ((p = va_arg(args, const char *))) - strvec_push(&cmd.args, p); + strvec_push(&argv, p); va_end(args); - cmd.git_cmd = 1; - return run_command(&cmd); + res = run_git_argv(&argv); + + strvec_clear(&argv); + return res; +} + +static const char *ensure_absolute_path(const char *path, char **absolute) +{ + struct strbuf buf = STRBUF_INIT; + + if (is_absolute_path(path)) + return path; + + strbuf_realpath_forgiving(&buf, path, 1); + free(*absolute); + *absolute = strbuf_detach(&buf, NULL); + return *absolute; } struct scalar_config { @@ -141,8 +202,10 @@ static int set_recommended_config(int reconfigure) { "commitGraph.changedPaths", "true" }, { "commitGraph.generationVersion", "1" }, { "core.autoCRLF", "false" }, + { "core.configLockTimeout", "150" }, { "core.logAllRefUpdates", "true" }, { "core.safeCRLF", "false" }, + { "core.untrackedCache", "true" }, { "credential.https://dev.azure.com.useHttpPath", "true" }, { "feature.experimental", "false" }, { "feature.manyFiles", "false" }, @@ -162,23 +225,7 @@ static int set_recommended_config(int reconfigure) { "status.aheadBehind", "false" }, /* platform-specific */ -#ifndef WIN32 - { "core.untrackedCache", "true" }, -#else - /* - * Unfortunately, Scalar's Functional Tests demonstrated - * that the untracked cache feature is unreliable on Windows - * (which is a bummer because that platform would benefit the - * most from it). For some reason, freshly created files seem - * not to update the directory's `lastModified` time - * immediately, but the untracked cache would need to rely on - * that. - * - * Therefore, with a sad heart, we disable this very useful - * feature on Windows. - */ - { "core.untrackedCache", "false" }, - +#ifdef WIN32 /* Other Windows-specific required settings: */ { "http.sslBackend", "schannel" }, #endif @@ -187,6 +234,45 @@ static int set_recommended_config(int reconfigure) int i; char *value; + /* + * If a user has "core.configWriteLockTimeoutMS" set, try to switch to + * the new (non-deprecated) setting (core.configLockTimeout). + */ + if (!repo_config_get_string(the_repository, "core.configwritelocktimeoutms", + &value)) { + char *dummy = NULL; + if (repo_config_get_string(the_repository, "core.configlocktimeout", + &dummy) && + repo_config_set_gently(the_repository, "core.configlocktimeout", + value)) + return error(_("could not configure %s=%s"), + "core.configLockTimeout", value); + if (repo_config_set_gently(the_repository, + "core.configwritelocktimeoutms", + NULL)) + return error(_("could not configure %s=%s"), + "core.configWriteLockTimeoutMS", "NULL"); + free(value); + free(dummy); + } + + /* + * If a user has "core.usebuiltinfsmonitor" enabled, try to switch to + * the new (non-deprecated) setting (core.fsmonitor). + */ + if (!repo_config_get_string(the_repository, "core.usebuiltinfsmonitor", &value)) { + char *dummy = NULL; + if (repo_config_get_string(the_repository, "core.fsmonitor", &dummy) && + repo_config_set_gently(the_repository, "core.fsmonitor", value) < 0) + return error(_("could not configure %s=%s"), + "core.fsmonitor", value); + if (repo_config_set_gently(the_repository, "core.usebuiltinfsmonitor", NULL) < 0) + return error(_("could not configure %s=%s"), + "core.useBuiltinFSMonitor", "NULL"); + free(value); + free(dummy); + } + for (i = 0; config[i].key; i++) { if (set_config_if_missing(config + i, reconfigure)) return error(_("could not configure %s=%s"), @@ -200,6 +286,33 @@ static int set_recommended_config(int reconfigure) fsmonitor.key, fsmonitor.value); } + /* + * Set HTTP/1.1 for Azure DevOps URLs + * We check for dev.azure.com/ and .visualstudio.com/ patterns + * which are sufficient to identify ADO URLs (including formats like + * https://orgname@dev.azure.com/...) + */ + if (!repo_config_get_string(the_repository, "remote.origin.url", &value)) { + if (starts_with(value, "https://dev.azure.com/") || + strstr(value, "@dev.azure.com/") || + strstr(value, ".visualstudio.com/")) { + struct strbuf key = STRBUF_INIT; + strbuf_addf(&key, "http.%s.version", value); + FREE_AND_NULL(value); + + if (reconfigure || repo_config_get_string(the_repository, key.buf, &value)) { + trace2_data_string("scalar", the_repository, key.buf, "created"); + if (repo_config_set_gently(the_repository, key.buf, "HTTP/1.1") < 0) { + strbuf_release(&key); + return error(_("could not configure %s=%s"), + key.buf, "HTTP/1.1"); + } + } + strbuf_release(&key); + } + FREE_AND_NULL(value); + } + /* * The `log.excludeDecoration` setting is special because it allows * for multiple values. @@ -338,6 +451,222 @@ static int set_config(const char *fmt, ...) return res; } +static int list_cache_server_urls(struct json_iterator *it) +{ + const char *p; + char *q; + long l; + + if (it->type == JSON_STRING && + skip_iprefix(it->key.buf, ".CacheServers[", &p) && + (l = strtol(p, &q, 10)) >= 0 && p != q && + !strcasecmp(q, "].Url")) + printf("#%ld: %s\n", l, it->string_value.buf); + + return 0; +} + +/* Find N for which .CacheServers[N].GlobalDefault == true */ +static int get_cache_server_index(struct json_iterator *it) +{ + const char *p; + char *q; + long l; + + if (it->type == JSON_TRUE && + skip_iprefix(it->key.buf, ".CacheServers[", &p) && + (l = strtol(p, &q, 10)) >= 0 && p != q && + !strcasecmp(q, "].GlobalDefault")) { + *(long *)it->fn_data = l; + return 1; + } + + return 0; +} + +struct cache_server_url_data { + char *key, *url; +}; + +/* Get .CacheServers[N].Url */ +static int get_cache_server_url(struct json_iterator *it) +{ + struct cache_server_url_data *data = it->fn_data; + + if (it->type == JSON_STRING && + !strcasecmp(data->key, it->key.buf)) { + data->url = strbuf_detach(&it->string_value, NULL); + return 1; + } + + return 0; +} + +static int can_url_support_gvfs(const char *url) +{ + return starts_with(url, "https://") || + (git_env_bool("GIT_TEST_ALLOW_GVFS_VIA_HTTP", 0) && + starts_with(url, "http://")); +} + +/* + * If `cache_server_url` is `NULL`, print the list to `stdout`. + * + * Since `gvfs-helper` requires a Git directory, this _must_ be run in + * a worktree. + */ +static int supports_gvfs_protocol(const char *url, char **cache_server_url) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + + /* + * The GVFS protocol is only supported via https://; For testing, we + * also allow http://. + */ + if (!can_url_support_gvfs(url)) + return 0; + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "-c", "http.version=HTTP/1.1", + "gvfs-helper", "--remote", url, "config", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { + long l = 0; + struct json_iterator it = + JSON_ITERATOR_INIT(out.buf, get_cache_server_index, &l); + struct cache_server_url_data data = { .url = NULL }; + + if (!cache_server_url) { + it.fn = list_cache_server_urls; + if (iterate_json(&it) < 0) { + reset_iterator(&it); + strbuf_release(&out); + return error("JSON parse error"); + } + reset_iterator(&it); + strbuf_release(&out); + return 0; + } + + if (iterate_json(&it) < 0) { + reset_iterator(&it); + strbuf_release(&out); + return error("JSON parse error"); + } + data.key = xstrfmt(".CacheServers[%ld].Url", l); + it.fn = get_cache_server_url; + it.fn_data = &data; + if (iterate_json(&it) < 0) { + reset_iterator(&it); + strbuf_release(&out); + return error("JSON parse error"); + } + *cache_server_url = data.url; + free(data.key); + reset_iterator(&it); + strbuf_release(&out); + return 1; + } + strbuf_release(&out); + /* error out quietly, unless we wanted to list URLs */ + return cache_server_url ? + 0 : error(_("Could not access gvfs/config endpoint")); +} + +static char *default_cache_root(const char *root) +{ + const char *env; + + if (is_unattended()) { + struct strbuf path = STRBUF_INIT; + strbuf_addstr(&path, root); + strip_last_path_component(&path); + strbuf_addstr(&path, "/.scalarCache"); + return strbuf_detach(&path, NULL); + } + +#ifdef WIN32 + (void)env; + return xstrfmt("%.*s.scalarCache", offset_1st_component(root), root); +#elif defined(__APPLE__) + if ((env = getenv("HOME")) && *env) + return xstrfmt("%s/.scalarCache", env); + return NULL; +#else + if ((env = getenv("XDG_CACHE_HOME")) && *env) + return xstrfmt("%s/scalar", env); + if ((env = getenv("HOME")) && *env) + return xstrfmt("%s/.cache/scalar", env); + return NULL; +#endif +} + +static int get_repository_id(struct json_iterator *it) +{ + if (it->type == JSON_STRING && + !strcasecmp(".repository.id", it->key.buf)) { + *(char **)it->fn_data = strbuf_detach(&it->string_value, NULL); + return 1; + } + + return 0; +} + +/* Needs to run this in a worktree; gvfs-helper requires a Git repository */ +static char *get_cache_key(const char *url) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + char *cache_key = NULL; + + /* + * The GVFS protocol is only supported via https://; For testing, we + * also allow http://. + */ + if (!git_env_bool("SCALAR_TEST_SKIP_VSTS_INFO", 0) && + can_url_support_gvfs(url)) { + cp.git_cmd = 1; + strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, + "endpoint", "vsts/info", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { + char *id = NULL; + struct json_iterator it = + JSON_ITERATOR_INIT(out.buf, get_repository_id, + &id); + + if (iterate_json(&it) < 0) + warning("JSON parse error (%s)", out.buf); + else if (id) + cache_key = xstrfmt("id_%s", id); + free(id); + } + } + + if (!cache_key) { + struct strbuf downcased = STRBUF_INIT; + int hash_algo_index = hash_algo_by_name("sha1"); + const struct git_hash_algo *hash_algo = hash_algo_index < 0 ? + the_hash_algo : &hash_algos[hash_algo_index]; + struct git_hash_ctx ctx; + unsigned char hash[GIT_MAX_RAWSZ]; + + strbuf_addstr(&downcased, url); + strbuf_tolower(&downcased); + + git_hash_init(&ctx, hash_algo); + git_hash_update(&ctx, downcased.buf, downcased.len); + git_hash_final(hash, &ctx); + + strbuf_release(&downcased); + + cache_key = xstrfmt("url_%s", + hash_to_hex_algop(hash, hash_algo)); + } + + strbuf_release(&out); + return cache_key; +} + static char *remote_default_branch(const char *url) { struct child_process cp = CHILD_PROCESS_INIT; @@ -432,12 +761,57 @@ void load_builtin_commands(const char *prefix UNUSED, die("not implemented"); } +static int init_shared_object_cache(const char *url, + const char *local_cache_root) +{ + struct strbuf buf = STRBUF_INIT; + int res = 0; + char *cache_key = NULL, *shared_cache_path = NULL, *alternates = NULL; + + if (!(cache_key = get_cache_key(url))) { + res = error(_("could not determine cache key for '%s'"), url); + goto cleanup; + } + + shared_cache_path = xstrfmt("%s/%s", local_cache_root, cache_key); + if (set_config("gvfs.sharedCache=%s", shared_cache_path)) { + res = error(_("could not configure shared cache")); + goto cleanup; + } + + strbuf_addf(&buf, "%s/pack", shared_cache_path); + switch (safe_create_leading_directories(the_repository, buf.buf)) { + case SCLD_OK: case SCLD_EXISTS: + break; /* okay */ + default: + res = error_errno(_("could not initialize '%s'"), buf.buf); + goto cleanup; + } + + alternates = repo_git_path(the_repository, "objects/info/alternates"); + write_file(alternates, "%s\n", shared_cache_path); + + cleanup: + strbuf_release(&buf); + free(shared_cache_path); + free(cache_key); + free(alternates); + return res; +} + static int cmd_clone(int argc, const char **argv) { + int dummy = 0; const char *branch = NULL; char *branch_to_free = NULL; int full_clone = 0, single_branch = 0, show_progress = isatty(2); - int src = 1, tags = 1, maintenance = 1; + int src = 1, tags = 1, maintenance = 1, prefetch = 1; + const char *cache_server_url = NULL, *local_cache_root = NULL; + char *default_cache_server_url = NULL, *local_cache_root_abs = NULL; + const char *prefetch_server = NULL, *get_server = NULL, *post_server = NULL; + int gvfs_protocol = -1; + const char *ref_format = NULL; + struct option clone_options[] = { OPT_STRING('b', "branch", &branch, N_(""), N_("branch to checkout after clone")), @@ -452,16 +826,44 @@ static int cmd_clone(int argc, const char **argv) N_("specify if tags should be fetched during clone")), OPT_BOOL(0, "maintenance", &maintenance, N_("specify if background maintenance should be enabled")), + OPT_BOOL(0, "prefetch", &prefetch, + N_("specify if commits and trees should be prefetched " + "during clone when using the GVFS Protocol")), + OPT_BOOL(0, "gvfs-protocol", &gvfs_protocol, + N_("force enable (or disable) the GVFS Protocol")), + OPT_STRING(0, "cache-server-url", &cache_server_url, + N_(""), + N_("the url or friendly name of the cache server")), + OPT_STRING(0, "prefetch-cache-server-url", &prefetch_server, + N_(""), + N_("the url or friendly name of a cache server for the prefetch endpoint")), + OPT_STRING(0, "get-cache-server-url", &get_server, + N_(""), + N_("the url or friendly name of a cache server for the objects GET endpoint")), + OPT_STRING(0, "post-cache-server-url", &post_server, + N_(""), + N_("the url or friendly name of a cache server for the objects POST endpoint")), + OPT_STRING(0, "local-cache-path", &local_cache_root, + N_(""), + N_("override the path for the local Scalar cache")), + OPT_STRING(0, "ref-format", &ref_format, N_("format"), + N_("specify the reference format to use")), + OPT_HIDDEN_BOOL(0, "no-fetch-commits-and-trees", + &dummy, N_("no longer used")), OPT_END(), }; const char * const clone_usage[] = { N_("scalar clone [--single-branch] [--branch ] [--full-clone]\n" - "\t[--[no-]src] [--[no-]tags] [--[no-]maintenance] []"), + "\t[--[no-]src] [--[no-]tags] [--[no-]maintenance] [--[no-]prefetch]\n" + "\t[--ref-format ]\n" + "\t[--cache-server-url ] [--[verb]-cache-server-url ]\n" + "\t[--local-cache-path ] []"), NULL }; const char *url; char *enlistment = NULL, *dir = NULL; struct strbuf buf = STRBUF_INIT; + struct strvec init_argv = STRVEC_INIT; int res; argc = parse_options(argc, argv, NULL, clone_options, clone_usage, 0); @@ -492,21 +894,43 @@ static int cmd_clone(int argc, const char **argv) if (is_directory(enlistment)) die(_("directory '%s' exists already"), enlistment); + ensure_absolute_path(enlistment, &enlistment); + if (src) dir = xstrfmt("%s/src", enlistment); else dir = xstrdup(enlistment); - strbuf_reset(&buf); + if (!local_cache_root) + local_cache_root = local_cache_root_abs = + default_cache_root(enlistment); + else + local_cache_root = ensure_absolute_path(local_cache_root, + &local_cache_root_abs); + + if (!local_cache_root) + die(_("could not determine local cache root")); + + strvec_clear(&init_argv); + strvec_pushf(&init_argv, "-c"); if (branch) - strbuf_addf(&buf, "init.defaultBranch=%s", branch); + strvec_pushf(&init_argv, "init.defaultBranch=%s", branch); else { char *b = repo_default_branch_name(the_repository, 1); - strbuf_addf(&buf, "init.defaultBranch=%s", b); + strvec_pushf(&init_argv, "init.defaultBranch=%s", b); free(b); } - if ((res = run_git("-c", buf.buf, "init", "--", dir, NULL))) + strvec_push(&init_argv, "init"); + + if (ref_format) { + strvec_push(&init_argv, "--ref-format"); + strvec_push(&init_argv, ref_format); + } + + strvec_push(&init_argv, "--"); + strvec_push(&init_argv, dir); + if ((res = run_git_argv(&init_argv))) goto cleanup; if (chdir(dir) < 0) { @@ -516,8 +940,28 @@ static int cmd_clone(int argc, const char **argv) setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + + /* + * This `dir_inside_of()` call relies on git_config() having parsed the + * newly-initialized repository config's `core.ignoreCase` value. + */ + if (dir_inside_of(local_cache_root, dir) >= 0) { + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, enlistment); + if (chdir("../..") < 0 || + remove_dir_recursively(&path, 0) < 0) + die(_("'--local-cache-path' cannot be inside the src " + "folder;\nCould not remove '%s'"), enlistment); + + die(_("'--local-cache-path' cannot be inside the src folder")); + } + /* common-main already logs `argv` */ trace2_def_repo(the_repository); + trace2_data_intmax("scalar", the_repository, "unattended", + is_unattended()); if (!branch && !(branch = branch_to_free = remote_default_branch(url))) { res = error(_("failed to get default branch for '%s'"), url); @@ -528,9 +972,7 @@ static int cmd_clone(int argc, const char **argv) set_config("remote.origin.fetch=" "+refs/heads/%s:refs/remotes/origin/%s", single_branch ? branch : "*", - single_branch ? branch : "*") || - set_config("remote.origin.promisor=true") || - set_config("remote.origin.partialCloneFilter=blob:none")) { + single_branch ? branch : "*")) { res = error(_("could not configure remote in '%s'"), dir); goto cleanup; } @@ -540,6 +982,80 @@ static int cmd_clone(int argc, const char **argv) goto cleanup; } + if (set_config("credential.https://dev.azure.com.useHttpPath=true")) { + res = error(_("could not configure credential.useHttpPath")); + goto cleanup; + } + + /* Is --[no-]gvfs-protocol unspecified? Infer from url. */ + if (gvfs_protocol < 0) { + if (cache_server_url || + strstr(url, "dev.azure.com/") || + strstr(url, "visualstudio.com")) + gvfs_protocol = 1; + else + gvfs_protocol = 0; + } + + if (gvfs_protocol && !supports_gvfs_protocol(url, &default_cache_server_url)) + die(_("failed to contact server via GVFS Protocol")); + + if (gvfs_protocol) { + if ((res = init_shared_object_cache(url, local_cache_root))) + goto cleanup; + if (!cache_server_url) + cache_server_url = default_cache_server_url; + if (set_config("core.useGVFSHelper=true") || + set_config("core.gvfs=%d", SCALAR_GVFS_MODE) || + set_config("http.%s.version=HTTP/1.1", url)) { + res = error(_("could not turn on GVFS helper")); + goto cleanup; + } + if (cache_server_url && + set_config("gvfs.cache-server=%s", cache_server_url)) { + res = error(_("could not configure cache server")); + goto cleanup; + } + if (cache_server_url) + fprintf(stderr, "Cache server URL: %s\n", + cache_server_url); + + if (prefetch_server && + set_config("gvfs.prefetch.cache-server=%s", prefetch_server)) { + res = error(_("could not configure prefetch cache server")); + goto cleanup; + } + if (prefetch_server) + fprintf(stderr, "Prefetch cache server URL: %s\n", + prefetch_server); + + if (get_server && + set_config("gvfs.get.cache-server=%s", get_server)) { + res = error(_("could not configure objects GET cache server")); + goto cleanup; + } + if (get_server) + fprintf(stderr, "Objects GET cache server URL: %s\n", + get_server); + + if (post_server && + set_config("gvfs.post.cache-server=%s", post_server)) { + res = error(_("could not configure objects POST cache server")); + goto cleanup; + } + if (post_server) + fprintf(stderr, "Objects POST cache server URL: %s\n", + post_server); + } else { + if (set_config("core.useGVFSHelper=false") || + set_config("remote.origin.promisor=true") || + set_config("remote.origin.partialCloneFilter=blob:none")) { + res = error(_("could not configure partial clone in " + "'%s'"), dir); + goto cleanup; + } + } + if (!full_clone && (res = run_git("sparse-checkout", "init", "--cone", NULL))) goto cleanup; @@ -547,11 +1063,34 @@ static int cmd_clone(int argc, const char **argv) if (set_recommended_config(0)) return error(_("could not configure '%s'"), dir); - if ((res = run_git("fetch", "--quiet", - show_progress ? "--progress" : "--no-progress", - "origin", - (tags ? NULL : "--no-tags"), - NULL))) { + strvec_clear(&init_argv); + /* + * When cloning with the GVFS Protocol, the `core.gvfs` value set + * above enables the GVFS_PREFETCH_DURING_FETCH bit, so the `git fetch` + * below issues a `/gvfs/prefetch` request to hydrate the local object + * cache. With `--no-prefetch`, skip that request for this initial + * fetch only (by clearing that bit for this invocation) so the + * worktree becomes ready sooner. The persisted `core.gvfs` value is + * left untouched, so subsequent fetches -- including background + * maintenance -- still prefetch as usual. + */ + if (gvfs_protocol && !prefetch) { + strvec_push(&init_argv, "-c"); + strvec_pushf(&init_argv, "core.gvfs=%d", + SCALAR_GVFS_MODE & ~GVFS_PREFETCH_DURING_FETCH); + } + strvec_pushl(&init_argv, "fetch", "--quiet", + show_progress ? "--progress" : "--no-progress", + "origin", NULL); + if (!tags) + strvec_push(&init_argv, "--no-tags"); + + if ((res = run_git_argv(&init_argv))) { + if (gvfs_protocol) { + res = error(_("failed to prefetch commits and trees")); + goto cleanup; + } + warning(_("partial clone failed; attempting full clone")); if (set_config("remote.origin.promisor") || @@ -574,6 +1113,26 @@ static int cmd_clone(int argc, const char **argv) strbuf_reset(&buf); strbuf_addf(&buf, "origin/%s", branch); + if (gvfs_protocol && !prefetch) { + struct object_id checkout_oid; + enum gh_client__created ghc; + + /* + * A commit requested via the GVFS objects POST endpoint + * includes the trees needed to check it out. + */ + repo_config(the_repository, git_default_config, NULL); + if (repo_get_oid(the_repository, buf.buf, &checkout_oid)) { + res = error(_("could not resolve '%s'"), buf.buf); + goto cleanup; + } + gh_client__queue_oid(&checkout_oid); + if (gh_client__drain_queue(&ghc)) { + res = error(_("failed to download trees for '%s'"), + buf.buf); + goto cleanup; + } + } res = run_git("checkout", "-f", "-t", buf.buf, NULL); if (res) goto cleanup; @@ -586,6 +1145,9 @@ static int cmd_clone(int argc, const char **argv) free(enlistment); free(dir); strbuf_release(&buf); + strvec_clear(&init_argv); + free(default_cache_server_url); + free(local_cache_root_abs); return res; } @@ -607,6 +1169,8 @@ static int cmd_diagnose(int argc, const char **argv) setup_enlistment_directory(argc, argv, usage, options, &diagnostics_root); strbuf_addstr(&diagnostics_root, "/.scalarDiagnostics"); + /* Here, a failure should not repeat itself. */ + git_retries = 1; res = run_git("diagnose", "--mode=all", "-s", "%Y%m%d_%H%M%S", "-o", diagnostics_root.buf, NULL); @@ -820,6 +1384,7 @@ static int cmd_run(int argc, const char **argv) { "fetch", "prefetch" }, { "loose-objects", "loose-objects" }, { "pack-files", "incremental-repack" }, + { "cache-local-objects", "cache-local-objects" }, { NULL, NULL } }; struct strbuf buf = STRBUF_INIT; @@ -996,6 +1561,68 @@ static int cmd_version(int argc, const char **argv) return 0; } +static int cmd_cache_server(int argc, const char **argv) +{ + int get = 0; + const char *set = NULL, *list = NULL; + struct option options[] = { + OPT_CMDMODE(0, "get", &get, + N_("get the configured cache-server URL"), 1), + OPT_STRING(0, "set", &set, N_("URL"), + N_("configure the cache-server to use")), + OPT_STRING(0, "list", &list, N_("remote"), + N_("list the possible cache-server URLs")), + OPT_END(), + }; + const char * const usage[] = { + N_("scalar cache-server " + "[--get | --set | --list ] []"), + NULL + }; + int res = 0; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + if (get + !!set + !!list > 1) + usage_msg_opt(_("--get/--set/--list are mutually exclusive"), + usage, options); + + setup_enlistment_directory(argc, argv, usage, options, NULL); + + if (list) { + const char *name = list, *url = list; + + if (!strchr(list, '/')) { + struct remote *remote; + + /* Look up remote */ + remote = remote_get(list); + if (!remote) { + error("no such remote: '%s'", name); + return 1; + } + if (!remote->url.nr) { + return error(_("remote '%s' has no URLs"), + name); + } + url = remote->url.v[0]; + } + res = supports_gvfs_protocol(url, NULL); + } else if (set) { + res = set_config("gvfs.cache-server=%s", set); + } else { + char *url = NULL; + + printf("Using cache server: %s\n", + repo_config_get_string(the_repository, "gvfs.cache-server", &url) ? + "(undefined)" : url); + free(url); + } + + return !!res; +} + static struct { const char *name; int (*fn)(int, const char **); @@ -1010,6 +1637,7 @@ static struct { { "help", cmd_help }, { "version", cmd_version }, { "diagnose", cmd_diagnose }, + { "cache-server", cmd_cache_server }, { NULL, NULL}, }; @@ -1018,6 +1646,12 @@ int cmd_main(int argc, const char **argv) struct strbuf scalar_usage = STRBUF_INIT; int i; + if (is_unattended()) { + setenv("GIT_ASKPASS", "", 0); + setenv("GIT_TERMINAL_PROMPT", "false", 0); + git_config_push_parameter("credential.interactive=false"); + } + while (argc > 1 && *argv[1] == '-') { if (!strcmp(argv[1], "-C")) { if (argc < 3) @@ -1041,6 +1675,9 @@ int cmd_main(int argc, const char **argv) argv++; argc--; + if (!strcmp(argv[0], "config")) + argv[0] = "reconfigure"; + for (i = 0; builtins[i].name; i++) if (!strcmp(builtins[i].name, argv[0])) return !!builtins[i].fn(argc, argv); diff --git a/send-pack.c b/send-pack.c index c6711b0a02509c..e9956899774613 100644 --- a/send-pack.c +++ b/send-pack.c @@ -3,6 +3,7 @@ #include "commit.h" #include "date.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "odb.h" #include "pkt-line.h" @@ -44,13 +45,14 @@ int option_parse_push_signed(const struct option *opt, static void append_negative_object(struct repository *r, struct oid_array *haves, - const struct object_id *oid) + const struct object_id *oid, + int check_missing) { /* * The remote end may have advertised objects that we do not have in * our object database. Skip those, as we cannot use them as boundary. */ - if (!odb_has_object(r->objects, oid, 0)) + if (check_missing && !odb_has_object(r->objects, oid, 0)) return; oid_array_append(haves, oid); } @@ -66,6 +68,8 @@ static int pack_objects(struct repository *r, struct odb_generate_pack_options opts = ODB_GENERATE_PACK_OPTIONS_INIT; struct odb_pack_generator *generator; int rc; + int negative_ref_check = 0; + int check_missing; trace2_region_enter("send_pack", "pack_objects", r); @@ -75,6 +79,7 @@ static int pack_objects(struct repository *r, opts.progress = ODB_GENERATE_PACK_PROGRESS_VERBOSE; opts.shallow = is_repository_shallow(r); opts.disable_bitmaps = args->disable_bitmaps; + opts.no_reuse_delta = args->no_reuse_delta; /* * The pack is either written directly to the remote's descriptor, or, @@ -83,14 +88,33 @@ static int pack_objects(struct repository *r, */ opts.pack_fd = args->stateless_rpc ? -1 : fd; + /* + * Normally we omit a negative (exclusion) object that we do not have + * locally. The core.gvfs GVFS_MISSING_OK bit disables that check, so + * missing negatives are still fed to pack-objects. Under the GVFS + * protocol that is harmful: pack-objects treats each fed exclusion as + * an edge and lazily downloads every one that is absent while hunting + * for preferred delta bases -- one immediate object request per + * advertised ref. Setting gvfs.negativeRefCheck restores the vanilla + * behavior of omitting a negative object we do not have, using a + * non-fetching existence check (odb_has_object() with flags 0 implies + * OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT). + */ + repo_config_get_bool(r, "gvfs.negativeRefCheck", &negative_ref_check); + check_missing = negative_ref_check || + !gvfs_config_is_set(r, GVFS_MISSING_OK); + for (size_t i = 0; i < advertised->nr; i++) - append_negative_object(r, &opts.haves, &advertised->oid[i]); + append_negative_object(r, &opts.haves, &advertised->oid[i], + check_missing); for (size_t i = 0; i < negotiated->nr; i++) - append_negative_object(r, &opts.haves, &negotiated->oid[i]); + append_negative_object(r, &opts.haves, &negotiated->oid[i], + check_missing); while (refs) { if (!is_null_oid(&refs->old_oid)) - append_negative_object(r, &opts.haves, &refs->old_oid); + append_negative_object(r, &opts.haves, &refs->old_oid, + check_missing); if (!is_null_oid(&refs->new_oid)) oid_array_append(&opts.wants, &refs->new_oid); refs = refs->next; diff --git a/send-pack.h b/send-pack.h index 13850c98bb093a..2af7a6b790dd65 100644 --- a/send-pack.h +++ b/send-pack.h @@ -28,6 +28,7 @@ struct send_pack_args { force_update:1, use_thin_pack:1, use_ofs_delta:1, + no_reuse_delta:1, dry_run:1, /* One of the SEND_PACK_PUSH_CERT_* constants. */ push_cert:2, diff --git a/sequencer.c b/sequencer.c index e25ef5eb61f9e9..ca470e2a6dd8a5 100644 --- a/sequencer.c +++ b/sequencer.c @@ -775,7 +775,7 @@ static int do_recursive_merge(struct repository *r, o.branch2 = next ? next_label : "(empty tree)"; if (is_rebase_i(opts)) o.buffer_output = 2; - o.show_rename_progress = 1; + o.show_rename_progress = isatty(2); head_tree = repo_parse_tree_indirect(the_repository, head); if (!head_tree) @@ -2641,7 +2641,7 @@ static int read_and_refresh_cache(struct repository *r, * expand the sparse index. */ if (opts->strategy && strcmp(opts->strategy, "ort")) - ensure_full_index(r->index); + ensure_full_index_with_reason(r->index, "non-ort merge strategy"); return 0; } @@ -3019,7 +3019,7 @@ static int have_finished_the_last_pick(void) } } /* If there is only one line then we are done */ - eol = strchr(buf.buf, '\n'); + eol = strchr(buf.buf, '\n'); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand if (!eol || !eol[1]) ret = 1; @@ -3279,9 +3279,9 @@ static int read_populate_opts(struct replay_opts *opts) if (read_oneliner(&buf, rebase_path_allow_rerere_autoupdate(), READ_ONELINER_SKIP_IF_EMPTY)) { - if (!strcmp(buf.buf, "--rerere-autoupdate")) + if (!strcmp(buf.buf, "--rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_auto = RERERE_AUTOUPDATE; - else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) + else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_auto = RERERE_NOAUTOUPDATE; strbuf_reset(&buf); } @@ -3331,7 +3331,7 @@ static int read_populate_opts(struct replay_opts *opts) READ_ONELINER_SKIP_IF_EMPTY)) { const char *p = ctx->current_fixups.buf; ctx->current_fixup_count = 1; - while ((p = strchr(p, '\n'))) { + while ((p = strchr(p, '\n'))) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand /* * Older versions of git accidentally * inserted blank lines when a fixup diff --git a/shallow.c b/shallow.c index 8e244a5669ec1f..2580ef2859f61d 100644 --- a/shallow.c +++ b/shallow.c @@ -755,7 +755,8 @@ void assign_shallow_commits_to_refs(struct shallow_info *info, for (i = 0; i < nr_shallow; i++) { struct commit *c = lookup_commit(the_repository, &oid[shallow[i]]); - c->object.flags |= BOTTOM; + if (c) + c->object.flags |= BOTTOM; } for (i = 0; i < ref->nr; i++) diff --git a/sparse-index.c b/sparse-index.c index 3d77dadae56d08..e21a91c47649c3 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -281,7 +281,7 @@ static int add_path_to_index(const struct object_id *oid, size_t len = base->len; if (S_ISDIR(mode)) { - int dtype; + int dtype = DT_DIR; size_t baselen = base->len; if (!ctx->pl) return READ_TREE_RECURSIVE; @@ -386,6 +386,10 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) full = xcalloc(1, sizeof(struct index_state)); memcpy(full, istate, sizeof(struct index_state)); + full->name_hash_initialized = 0; + memset(&full->name_hash, 0, sizeof(full->name_hash)); + memset(&full->dir_hash, 0, sizeof(full->dir_hash)); + /* * This slightly-misnamed 'full' index might still be sparse if we * are only modifying the list of sparse directories. This hinges @@ -405,7 +409,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) struct cache_entry *ce = istate->cache[i]; struct tree *tree; struct pathspec ps; - int dtype; + int dtype = DT_UNKNOWN; if (!S_ISSPARSEDIR(ce->ce_mode)) { set_index_entry(full, full->cache_nr++, ce); @@ -416,7 +420,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) if (pl && path_matches_pattern_list(ce->name, ce->ce_namelen, NULL, &dtype, - pl, istate) == NOT_MATCHED) { + pl, full) == NOT_MATCHED) { set_index_entry(full, full->cache_nr++, ce); continue; } @@ -444,8 +448,15 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) } /* Copy back into original index. */ + if (istate->name_hash_initialized) { + hashmap_clear(&istate->name_hash); + hashmap_clear(&istate->dir_hash); + } + + istate->name_hash_initialized = full->name_hash_initialized; memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash)); memcpy(&istate->dir_hash, &full->dir_hash, sizeof(full->dir_hash)); + istate->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED; free(istate->cache); istate->cache = full->cache; @@ -473,6 +484,24 @@ void ensure_full_index(struct index_state *istate) expand_index(istate, NULL); } +void ensure_full_index_with_reason(struct index_state *istate, + const char *fmt, ...) +{ + va_list ap; + struct strbuf why = STRBUF_INIT; + if (!istate) + BUG("ensure_full_index_with_reason() must get an index!"); + if (istate->sparse_index == INDEX_EXPANDED) + return; + + va_start(ap, fmt); + strbuf_vaddf(&why, fmt, ap); + trace2_data_string("sparse-index", istate->repo, "expansion-reason", why.buf); + va_end(ap); + strbuf_release(&why); + ensure_full_index(istate); +} + void ensure_correct_sparsity(struct index_state *istate) { /* @@ -482,7 +511,8 @@ void ensure_correct_sparsity(struct index_state *istate) if (is_sparse_index_allowed(istate, 0)) convert_to_sparse(istate, 0); else - ensure_full_index(istate); + ensure_full_index_with_reason(istate, + "sparse index not allowed"); } struct path_found_data { @@ -630,6 +660,8 @@ static int clear_skip_worktree_from_present_files_sparse(struct index_state *ist if (path_found(ce->name, &data)) { if (S_ISSPARSEDIR(ce->ce_mode)) { to_restart = 1; + trace2_data_string("sparse-index", istate->repo, + "skip-worktree sparsedir", ce->name); break; } ce->ce_flags &= ~CE_SKIP_WORKTREE; @@ -682,11 +714,13 @@ void clear_skip_worktree_from_present_files(struct index_state *istate) struct repo_config_values *cfg = repo_config_values(the_repository); if (!cfg->apply_sparse_checkout || + core_virtualfilesystem || cfg->sparse_expect_files_outside_of_patterns) return; if (clear_skip_worktree_from_present_files_sparse(istate)) { - ensure_full_index(istate); + ensure_full_index_with_reason(istate, + "failed to clear skip-worktree while sparse"); clear_skip_worktree_from_present_files_full(istate); } } @@ -749,7 +783,9 @@ void expand_to_path(struct index_state *istate, * in the index, perhaps it exists within this * sparse-directory. Expand accordingly. */ - ensure_full_index(istate); + const char *fmt = "found index entry for '%s'"; + ensure_full_index_with_reason(istate, fmt, + path_mutable.buf); break; } diff --git a/sparse-index.h b/sparse-index.h index 727034be7ca917..15180b02ea6599 100644 --- a/sparse-index.h +++ b/sparse-index.h @@ -1,6 +1,8 @@ #ifndef SPARSE_INDEX_H__ #define SPARSE_INDEX_H__ +#include "strbuf.h" + /* * If performing an operation where the index is supposed to expand to a * full index, then disable the advice message by setting this global to @@ -46,4 +48,16 @@ void expand_index(struct index_state *istate, struct pattern_list *pl); void ensure_full_index(struct index_state *istate); +/** + * If there is a clear reason why the sparse index is being expanded, then + * trace the information for why the expansion is occurring. + */ +void ensure_full_index_with_reason(struct index_state *istate, + const char *fmt, + ...); + +#define ensure_full_index_unaudited(i) \ + ensure_full_index_with_reason((i), \ + "unaudited call (%s.%d)", __FILE__, __LINE__); + #endif diff --git a/src/cargo-meson.sh b/src/cargo-meson.sh index 83c7e7b79b9d6f..92efbd03842d9a 100755 --- a/src/cargo-meson.sh +++ b/src/cargo-meson.sh @@ -22,7 +22,11 @@ done case "$(cargo -vV | sed -n 's/^host: \(.*\)$/\1/p')" in *-windows-msvc) LIBNAME=gitcore.lib - PATH="$(echo "$PATH" | tr ':' '\n' | grep -Ev "^(/mingw64/bin|/usr/bin)$" | paste -sd: -):/mingw64/bin:/usr/bin" + mingw_prefix=${MINGW_PREFIX:-/$(printf '%s' \ + "${MSYSTEM:-UCRT64}" | tr A-Z a-z)} + PATH="$(echo "$PATH" | tr ':' '\n' | + grep -Fxv -e "$mingw_prefix/bin" -e /usr/bin | + paste -sd: -):$mingw_prefix/bin:/usr/bin" export PATH ;; *-windows-*) diff --git a/strvec.c b/strvec.c index f8de79f5579b49..6f0ec491a4d05a 100644 --- a/strvec.c +++ b/strvec.c @@ -22,7 +22,7 @@ void strvec_push_nodup(struct strvec *array, char *value) const char *strvec_push(struct strvec *array, const char *value) { - strvec_push_nodup(array, xstrdup(value)); + strvec_push_nodup(array, xstrdup(value)); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand return array->v[array->nr - 1]; } diff --git a/sub-process.c b/sub-process.c index 2d5c965169727b..d1c0bb4cca5cd3 100644 --- a/sub-process.c +++ b/sub-process.c @@ -5,6 +5,7 @@ #include "sub-process.h" #include "sigchain.h" #include "pkt-line.h" +#include "quote.h" int cmd2process_cmp(const void *cmp_data UNUSED, const struct hashmap_entry *eptr, @@ -59,6 +60,8 @@ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) finish_command(&entry->process); hashmap_remove(hashmap, &entry->ent, NULL); + FREE_AND_NULL(entry->to_free); + entry->cmd = NULL; } static void subprocess_exit_handler(struct child_process *process) @@ -78,7 +81,12 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co int err; struct child_process *process; - entry->cmd = cmd; + // BUGBUG most callers to subprocess_start() pass in "cmd" the value + // BUGBUG of find_hook() which returns a static buffer (that's only + // BUGBUG good until the next call to find_hook()). + // BUGFIX Defer assignment until we copy the string in our argv. + // entry->cmd = cmd; + process = &entry->process; child_process_init(process); @@ -90,6 +98,9 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co process->clean_on_exit_handler = subprocess_exit_handler; process->trace2_child_class = "subprocess"; + entry->cmd = process->args.v[0]; + entry->to_free = NULL; + err = start_command(process); if (err) { error("cannot fork to run subprocess '%s'", cmd); @@ -109,6 +120,54 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co return 0; } +int subprocess_start_strvec(struct hashmap *hashmap, + struct subprocess_entry *entry, + int is_git_cmd, + const struct strvec *argv, + subprocess_start_fn startfn) +{ + int err; + struct child_process *process; + struct strbuf quoted = STRBUF_INIT; + + process = &entry->process; + + child_process_init(process); + strvec_pushv(&process->args, argv->v); + process->use_shell = 1; + process->in = -1; + process->out = -1; + process->git_cmd = is_git_cmd; + process->clean_on_exit = 1; + process->clean_on_exit_handler = subprocess_exit_handler; + process->trace2_child_class = "subprocess"; + + sq_quote_argv_pretty("ed, argv->v); + entry->cmd = entry->to_free = strbuf_detach("ed, NULL); + + err = start_command(process); + if (err) { + error("cannot fork to run subprocess '%s'", entry->cmd); + FREE_AND_NULL(entry->to_free); + entry->cmd = NULL; + return err; + } + + hashmap_entry_init(&entry->ent, strhash(entry->cmd)); + + err = startfn(entry); + if (err) { + error("initialization for subprocess '%s' failed", entry->cmd); + FREE_AND_NULL(entry->to_free); + entry->cmd = NULL; + subprocess_stop(hashmap, entry); + return err; + } + + hashmap_add(hashmap, &entry->ent); + return 0; +} + static int handshake_version(struct child_process *process, const char *welcome_prefix, int *versions, int *chosen_version) diff --git a/sub-process.h b/sub-process.h index bfc3959a1b4894..a5dae83af5b138 100644 --- a/sub-process.h +++ b/sub-process.h @@ -25,6 +25,12 @@ struct subprocess_entry { struct hashmap_entry ent; const char *cmd; + /** + * In case `cmd` is a `strdup()`ed value that needs to be released, + * you can assign the pointer to `to_free` so that `subprocess_stop()` + * will release it. + */ + char *to_free; struct child_process process; }; @@ -56,6 +62,12 @@ typedef int(*subprocess_start_fn)(struct subprocess_entry *entry); int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn); +int subprocess_start_strvec(struct hashmap *hashmap, + struct subprocess_entry *entry, + int is_git_cmd, + const struct strvec *argv, + subprocess_start_fn startfn); + /* Kill a subprocess and remove it from the subprocess hashmap. */ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry); diff --git a/submodule.c b/submodule.c index bbd86f82a9ccbb..fbcb3e51b5270d 100644 --- a/submodule.c +++ b/submodule.c @@ -2664,11 +2664,11 @@ int get_superproject_working_tree(struct strbuf *buf) * The format is SP SP TAB \0, * We're only interested in the name after the tab. */ - super_sub = strchr(sb.buf, '\t') + 1; - super_sub_len = strlen(super_sub); + super_sub = strchr(sb.buf, '\t') + 1; // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand + super_sub_len = strlen(super_sub); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand if (super_sub_len > cwd_len || - strcmp(&cwd[cwd_len - super_sub_len], super_sub)) + strcmp(&cwd[cwd_len - super_sub_len], super_sub)) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand BUG("returned path string doesn't match cwd?"); super_wt = xstrdup(cwd); diff --git a/t/helper/.gitignore b/t/helper/.gitignore index 8c2ddcce95f7aa..4687ed470c5978 100644 --- a/t/helper/.gitignore +++ b/t/helper/.gitignore @@ -1,2 +1,3 @@ +/test-gvfs-protocol /test-tool /test-fake-ssh diff --git a/t/helper/meson.build b/t/helper/meson.build index d4499d26a9af1f..80ff7dc776cbff 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -89,6 +89,13 @@ test_tool = executable('test-tool', bin_wrappers += test_tool test_dependencies += test_tool +test_gvfs_protocol = executable('test-gvfs-protocol', + sources: 'test-gvfs-protocol.c', + dependencies: [libgit_commonmain], +) +bin_wrappers += test_gvfs_protocol +test_dependencies += test_gvfs_protocol + test_fake_ssh = executable('test-fake-ssh', sources: 'test-fake-ssh.c', dependencies: [libgit_commonmain], diff --git a/t/helper/test-gvfs-protocol.c b/t/helper/test-gvfs-protocol.c new file mode 100644 index 00000000000000..4665f31814d154 --- /dev/null +++ b/t/helper/test-gvfs-protocol.c @@ -0,0 +1,2351 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "environment.h" +#include "gettext.h" +#include "hex.h" +#include "alloc.h" +#include "setup.h" +#include "protocol.h" +#include "config.h" +#include "pkt-line.h" +#include "run-command.h" +#include "strbuf.h" +#include "string-list.h" +#include "trace2.h" +#include "copy.h" +#include "object.h" +#include "object-file.h" +#include "odb.h" +#include "replace-object.h" +#include "repository.h" +#include "version.h" +#include "dir.h" +#include "json-writer.h" +#include "oidset.h" +#include "date.h" +#include "wrapper.h" +#include "git-zlib.h" +#include "packfile.h" + +#define TR2_CAT "test-gvfs-protocol" + +static const char *pid_file; +static int verbose; +static int reuseaddr; +static struct string_list mayhem_list = STRING_LIST_INIT_DUP; +static int mayhem_child = 0; +static struct json_writer jw_config = JSON_WRITER_INIT; + +/* + * We look for one of these "servertypes" in the uri-base + * so we can behave differently when we need to. + */ +#define MY_SERVER_TYPE__ORIGIN "servertype/origin" +#define MY_SERVER_TYPE__CACHE "servertype/cache" + +static const char test_gvfs_protocol_usage[] = +"gvfs-protocol [--verbose]\n" +" [--timeout=] [--init-timeout=] [--max-connections=]\n" +" [--reuseaddr] [--pid-file=]\n" +" [--listen=]* [--port=]\n" +" [--mayhem=]*\n" +; + +/* Timeout, and initial timeout */ +static unsigned int timeout; +static unsigned int init_timeout; + +static void logreport(const char *label, const char *err, va_list params) +{ + struct strbuf msg = STRBUF_INIT; + + strbuf_addf(&msg, "[%"PRIuMAX"] %s: ", (uintmax_t)getpid(), label); + strbuf_vaddf(&msg, err, params); + strbuf_addch(&msg, '\n'); + + fwrite(msg.buf, sizeof(char), msg.len, stderr); + fflush(stderr); + + strbuf_release(&msg); +} + +__attribute__((format (printf, 1, 2))) +static void logerror(const char *err, ...) +{ + va_list params; + va_start(params, err); + logreport("error", err, params); + va_end(params); +} + +__attribute__((format (printf, 1, 2))) +static void loginfo(const char *err, ...) +{ + va_list params; + if (!verbose) + return; + va_start(params, err); + logreport("info", err, params); + va_end(params); +} + +__attribute__((format (printf, 1, 2))) +static void logmayhem(const char *err, ...) +{ + va_list params; + if (!verbose) + return; + va_start(params, err); + logreport("mayhem", err, params); + va_end(params); +} + +static void set_keep_alive(int sockfd) +{ + int ka = 1; + + if (setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &ka, sizeof(ka)) < 0) { + if (errno != ENOTSOCK) + logerror("unable to set SO_KEEPALIVE on socket: %s", + strerror(errno)); + } +} + +////////////////////////////////////////////////////////////////// +// The code in this section is used by "worker" instances to service +// a single connection from a client. The worker talks to the client +// on 0 and 1. +////////////////////////////////////////////////////////////////// + +enum worker_result { + /* + * Operation successful. + * Caller *might* keep the socket open and allow keep-alive. + */ + WR_OK = 0, + /* + * Various errors while processing the request and/or the response. + * Close the socket and clean up. + * Exit child-process with non-zero status. + */ + WR_IO_ERROR = 1<<0, + /* + * Close the socket and clean up. Does not imply an error. + */ + WR_HANGUP = 1<<1, + /* + * The result of a function was influenced by the mayhem settings. + * Does not imply that we need to exit or close the socket. + * Just advice to callers in the worker stack. + */ + WR_MAYHEM = 1<<2, + + WR_STOP_THE_MUSIC = (WR_IO_ERROR | WR_HANGUP), +}; + +/* + * Fields from a parsed HTTP request. + */ +struct req { + struct strbuf start_line; + struct string_list start_line_fields; + + struct strbuf uri_base; + struct strbuf gvfs_api; + struct strbuf slash_args; + struct strbuf quest_args; + + struct string_list header_list; +}; + +#define REQ__INIT { \ + .start_line = STRBUF_INIT, \ + .start_line_fields = STRING_LIST_INIT_DUP, \ + .uri_base = STRBUF_INIT, \ + .gvfs_api = STRBUF_INIT, \ + .slash_args = STRBUF_INIT, \ + .quest_args = STRBUF_INIT, \ + .header_list = STRING_LIST_INIT_DUP, \ + } + +static void req__release(struct req *req) +{ + strbuf_release(&req->start_line); + string_list_clear(&req->start_line_fields, 0); + + strbuf_release(&req->uri_base); + strbuf_release(&req->gvfs_api); + strbuf_release(&req->slash_args); + strbuf_release(&req->quest_args); + + string_list_clear(&req->header_list, 0); +} + +/* + * Generate a somewhat bogus UUID/GUID that is good enough for + * a test suite, but without requiring platform-specific UUID + * or GUID libraries. + */ +static void gen_fake_uuid(struct strbuf *uuid) +{ + static unsigned int seq = 0; + static struct timeval tv; + static struct tm tm; + static time_t secs; + + strbuf_setlen(uuid, 0); + + if (!seq) { + gettimeofday(&tv, NULL); + secs = tv.tv_sec; + gmtime_r(&secs, &tm); + } + + /* + * Build a string that looks like: + * + * "ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb" + * + * Note that the first digit in the "dddd" section gives the + * UUID type. We set it to zero so that we won't collide with + * any "real" UUIDs. + */ + strbuf_addf(uuid, "%04d%02d%02d-%02d%02d-00%02d-%04x-%08x%04x", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, + tm.tm_sec, + (unsigned)(getpid() & 0xffff), + (unsigned)(tv.tv_usec & 0xffffffff), + (seq++ & 0xffff)); +} + +/* + * Send a chunk of data to the client using HTTP chunked + * transfer coding rules. + * + * https://tools.ietf.org/html/rfc7230#section-4.1 + */ +static enum worker_result send_chunk(int fd, const unsigned char *buf, + size_t len_buf) +{ + char chunk_size[100]; + int chunk_size_len = xsnprintf(chunk_size, sizeof(chunk_size), + "%x\r\n", (unsigned int)len_buf); + + if ((write_in_full(fd, chunk_size, chunk_size_len) < 0) || + (write_in_full(fd, buf, len_buf) < 0) || + (write_in_full(fd, "\r\n", 2) < 0)) { + logerror("unable to send chunk"); + return WR_IO_ERROR; + } + + return WR_OK; +} + +static enum worker_result send_final_chunk(int fd) +{ + if (write_in_full(fd, "0\r\n\r\n", 5) < 0) { + logerror("unable to send final chunk"); + return WR_IO_ERROR; + } + + return WR_OK; +} + +static enum worker_result send_http_error( + int fd, + int http_code, const char *http_code_name, + int retry_after_seconds, enum worker_result wr_in) +{ + struct strbuf response_header = STRBUF_INIT; + struct strbuf response_content = STRBUF_INIT; + struct strbuf uuid = STRBUF_INIT; + enum worker_result wr; + + strbuf_addf(&response_content, "Error: %d %s\r\n", + http_code, http_code_name); + if (retry_after_seconds > 0) + strbuf_addf(&response_content, "Retry-After: %d\r\n", + retry_after_seconds); + + strbuf_addf (&response_header, "HTTP/1.1 %d %s\r\n", http_code, http_code_name); + strbuf_addstr(&response_header, "Cache-Control: private\r\n"); + strbuf_addstr(&response_header, "Content-Type: text/plain\r\n"); + strbuf_addf (&response_header, "Content-Length: %d\r\n", (int)response_content.len); + if (retry_after_seconds > 0) + strbuf_addf (&response_header, "Retry-After: %d\r\n", retry_after_seconds); + strbuf_addf( &response_header, "Server: test-gvfs-protocol/%s\r\n", git_version_string); + strbuf_addf( &response_header, "Date: %s\r\n", show_date(time(NULL), 0, DATE_MODE(RFC2822))); + gen_fake_uuid(&uuid); + strbuf_addf( &response_header, "X-VSS-E2EID: %s\r\n", uuid.buf); + strbuf_addstr(&response_header, "\r\n"); + + if (write_in_full(fd, response_header.buf, response_header.len) < 0) { + logerror("unable to write response header"); + wr = WR_IO_ERROR; + goto done; + } + + if (write_in_full(fd, response_content.buf, response_content.len) < 0) { + logerror("unable to write response content body"); + wr = WR_IO_ERROR; + goto done; + } + + wr = wr_in; + +done: + strbuf_release(&uuid); + strbuf_release(&response_header); + strbuf_release(&response_content); + + return wr; +} + +/* + * Return 1 if we send an AUTH error to the client. + */ +static int mayhem_try_auth(struct req *req, enum worker_result *wr_out) +{ + *wr_out = WR_OK; + + if (string_list_has_string(&mayhem_list, "http_401_1") && + mayhem_child == 0) { + logmayhem("http_401_1"); + *wr_out = send_http_error(1, 401, "Unauthorized", -1, + WR_MAYHEM); + return 1; + } + + if (string_list_has_string(&mayhem_list, "http_401")) { + struct string_list_item *item; + int has_auth = 0; + for_each_string_list_item(item, &req->header_list) { + if (starts_with(item->string, "Authorization: Basic")) { + has_auth = 1; + break; + } + } + if (!has_auth) { + if (strstr(req->uri_base.buf, MY_SERVER_TYPE__ORIGIN)) { + logmayhem("http_401 (origin)"); + *wr_out = send_http_error(1, 401, "Unauthorized", -1, + WR_MAYHEM); + return 1; + } + + else if (strstr(req->uri_base.buf, MY_SERVER_TYPE__CACHE)) { + /* + * Cache servers use a non-standard 400 rather than a 401. + */ + logmayhem("http_400 (cacheserver)"); + *wr_out = send_http_error(1, 400, "Bad Request", -1, + WR_MAYHEM); + return 1; + } + + else { + /* + * Non-qualified server type. + */ + logmayhem("http_401"); + *wr_out = send_http_error(1, 401, "Unauthorized", -1, + WR_MAYHEM); + return 1; + } + } + } + + return 0; +} + +/* + * Build fake gvfs/config data using our IP address and port. + * + * The Min/Max data is just random noise copied from the example + * in the documentation. + */ +static void build_gvfs_config_json(struct json_writer *jw, + struct string_list *listen_addr, + int listen_port) +{ + jw_object_begin(jw, 0); + { + jw_object_inline_begin_array(jw, "AllowedGvfsClientVersions"); + { + jw_array_inline_begin_object(jw); + { + jw_object_inline_begin_object(jw, "Max"); + { + jw_object_intmax(jw, "Major", 0); + jw_object_intmax(jw, "Minor", 4); + jw_object_intmax(jw, "Build", 0); + jw_object_intmax(jw, "Revision", 0); + } + jw_end(jw); + + jw_object_inline_begin_object(jw, "Min"); + { + jw_object_intmax(jw, "Major", 0); + jw_object_intmax(jw, "Minor", 2); + jw_object_intmax(jw, "Build", 0); + jw_object_intmax(jw, "Revision", 0); + } + jw_end(jw); + } + jw_end(jw); + + jw_array_inline_begin_object(jw); + { + jw_object_null(jw, "Max"); + jw_object_inline_begin_object(jw, "Min"); + { + jw_object_intmax(jw, "Major", 0); + jw_object_intmax(jw, "Minor", 5); + jw_object_intmax(jw, "Build", 16326); + jw_object_intmax(jw, "Revision", 1); + } + jw_end(jw); + } + jw_end(jw); + } + jw_end(jw); + + jw_object_inline_begin_array(jw, "CacheServers"); + { + struct string_list_item *item; + int k = 0; + + for_each_string_list_item(item, listen_addr) { + jw_array_inline_begin_object(jw); + { + struct strbuf buf = STRBUF_INIT; + + strbuf_addf(&buf, "http://%s:%d/%s", + item->string, + listen_port, + MY_SERVER_TYPE__CACHE); + jw_object_string(jw, "Url", buf.buf); + strbuf_release(&buf); + + strbuf_addf(&buf, "cs%02d", k); + jw_object_string(jw, "Name", buf.buf); + strbuf_release(&buf); + + jw_object_bool(jw, "GlobalDefault", + k++ == 0); + } + jw_end(jw); + } + } + jw_end(jw); + } + jw_end(jw); +} +/* + * Per the GVFS Protocol, this should only be recognized on the origin + * server (not the cache-server). It returns a JSON payload of config + * data. + */ +static enum worker_result do__gvfs_config__get(struct req *req) +{ + struct strbuf response_header = STRBUF_INIT; + struct strbuf uuid = STRBUF_INIT; + enum worker_result wr; + + if (strstr(req->uri_base.buf, MY_SERVER_TYPE__CACHE)) + return send_http_error(1, 404, "Not Found", -1, WR_OK); + + strbuf_addstr(&response_header, "HTTP/1.1 200 OK\r\n"); + strbuf_addstr(&response_header, "Cache-Control: private\r\n"); + strbuf_addstr(&response_header, "Content-Type: text/plain\r\n"); + strbuf_addf( &response_header, "Content-Length: %d\r\n", (int)jw_config.json.len); + strbuf_addf( &response_header, "Server: test-gvfs-protocol/%s\r\n", git_version_string); + strbuf_addf( &response_header, "Date: %s\r\n", show_date(time(NULL), 0, DATE_MODE(RFC2822))); + gen_fake_uuid(&uuid); + strbuf_addf( &response_header, "X-VSS-E2EID: %s\r\n", uuid.buf); + strbuf_addstr(&response_header, "\r\n"); + + if (write_in_full(1, response_header.buf, response_header.len) < 0) { + logerror("unable to write response header"); + wr = WR_IO_ERROR; + goto done; + } + + if (write_in_full(1, jw_config.json.buf, jw_config.json.len) < 0) { + logerror("unable to write response content body"); + wr = WR_IO_ERROR; + goto done; + } + + wr = WR_OK; + +done: + strbuf_release(&uuid); + strbuf_release(&response_header); + + return wr; +} + +/* + * Send the contents of the in-memory inflated object in "compressed + * loose object" format over the socket. + * + * Because we are using keep-alive and are streaming the compressed + * chunks as we produce them, we set the transport-encoding and not + * the content-length. + * + * Our usage here is different from `git-http-backend` because it will + * only send a loose object if it exists as a loose object in the ODB + * (see the "/objects/[0-9a-f]{2}/[0-9a-f]{38}$" regex_t declarations) + * by doing a file-copy. + * + * We want to send an arbitrary object without regard for how it is + * currently stored in the local ODB. + * + * Also, we don't want any of the type-specific branching found in the + * sha1-file.c functions (such as special casing BLOBs). Specifically, + * we DO NOT want any of the content conversion filters. We just want + * to send the raw content as is. + * + * So, we steal freely from sha1-file.c routines: + * write_object_file_prepare() + * write_loose_object() + */ +static enum worker_result send_loose_object(const struct object_id *oid, + int fd) +{ +#define MAX_HEADER_LEN 32 + struct strbuf response_header = STRBUF_INIT; + struct strbuf uuid = STRBUF_INIT; + char object_header[MAX_HEADER_LEN]; + unsigned char compressed[4096]; + git_zstream stream; + struct object_id oid_check; + struct git_hash_ctx c; + int object_header_len; + int ret; + struct repo_config_values *cfg; + unsigned flags = 0; + void *content; + size_t size; + enum object_type type; + struct object_info oi = OBJECT_INFO_INIT; + int mayhem__corrupt_loose = string_list_has_string(&mayhem_list, + "corrupt_loose"); + + /* + * Since `test-gvfs-protocol` is mocking a real GVFS server (cache or + * main), we don't want a request for a missing object to cause the + * implicit dynamic fetch mechanism to try to fault-it-in (and cause + * our call to odb_read_object_info_extended() to launch another instance + * of `gvfs-helper` to magically fetch it (which would connect to a + * new instance of `test-gvfs-protocol`)). + * + * Rather, we want a missing object to fail, so we can respond with + * a 404, for example. + */ + flags |= OBJECT_INFO_FOR_PREFETCH; + flags |= OBJECT_INFO_LOOKUP_REPLACE; + + oi.typep = &type; + oi.sizep = &size; + oi.contentp = &content; + + if (odb_read_object_info_extended(the_repository->objects, oid, &oi, flags)) { + logerror("Could not find OID: '%s'", oid_to_hex(oid)); + free(content); + return send_http_error(1, 404, "Not Found", -1, WR_OK); + } + + if (string_list_has_string(&mayhem_list, "http_404")) { + logmayhem("http_404"); + free(content); + return send_http_error(1, 404, "Not Found", -1, WR_MAYHEM); + } + + /* + * We are blending several somewhat independent concepts here: + * + * [1] reconstructing the object format in parts: + * + * ::= + * + * [1a] ::= SP NUL + * [1b] ::= + * + * [2] verify that we constructed [1] correctly by computing + * the hash of [1] and verify it matches the passed OID. + * + * [3] compress [1] because that is how loose objects are + * stored on disk. We compress it as we stream it to + * the client. + * + * [4] send HTTP response headers to the client. + * + * [5] stream each chunk from [3] to the client using the HTTP + * chunked transfer coding. + * + * [6] for extra credit, we repeat the hash construction in [2] + * as we stream it. + */ + + /* [4] */ + strbuf_addstr(&response_header, "HTTP/1.1 200 OK\r\n"); + strbuf_addstr(&response_header, "Cache-Control: private\r\n"); + strbuf_addstr(&response_header, "Content-Type: application/x-git-loose-object\r\n"); + strbuf_addf( &response_header, "Server: test-gvfs-protocol/%s\r\n", git_version_string); + strbuf_addstr(&response_header, "Transfer-Encoding: chunked\r\n"); + strbuf_addf( &response_header, "Date: %s\r\n", show_date(time(NULL), 0, DATE_MODE(RFC2822))); + gen_fake_uuid(&uuid); + strbuf_addf( &response_header, "X-VSS-E2EID: %s\r\n", uuid.buf); + strbuf_addstr(&response_header, "\r\n"); + + if (write_in_full(fd, response_header.buf, response_header.len) < 0) { + logerror("unable to write response header"); + free(content); + return WR_IO_ERROR; + } + + strbuf_release(&uuid); + strbuf_release(&response_header); + + if (string_list_has_string(&mayhem_list, "close_write")) { + logmayhem("close_write"); + free(content); + return WR_MAYHEM | WR_HANGUP; + } + + /* [1a] */ + object_header_len = 1 + xsnprintf(object_header, MAX_HEADER_LEN, + "%s %"PRIuMAX, + type_name(*oi.typep), + (uintmax_t)*oi.sizep); + + /* [2] */ + memset(&oid_check, 0, sizeof(oid_check)); + git_hash_init(&c, the_hash_algo); + git_hash_update(&c, object_header, object_header_len); + git_hash_update(&c, *oi.contentp, *oi.sizep); + git_hash_final(oid_check.hash, &c); + if (!oideq(oid, &oid_check)) + BUG("send_loose_object[2]: invalid construction '%s' '%s'", + oid_to_hex(oid), oid_to_hex(&oid_check)); + + /* [3, 6] */ + cfg = repo_config_values(the_repository); + git_deflate_init(&stream, cfg->zlib_compression_level); + stream.next_out = compressed; + stream.avail_out = sizeof(compressed); + git_hash_init(&c, the_hash_algo); + + /* [3, 1a, 6] */ + stream.next_in = (unsigned char *)object_header; + stream.avail_in = object_header_len; + while (git_deflate(&stream, 0) == Z_OK) + ; /* nothing */ + git_hash_update(&c, object_header, object_header_len); + + /* [3, 1b, 5, 6] */ + stream.next_in = *oi.contentp; + stream.avail_in = *oi.sizep; + do { + enum worker_result wr; + unsigned char *in0 = stream.next_in; + + /* + * Corrupt a byte in the buffer we compress, but undo it + * before we compute the SHA on the portion of the raw + * buffer included in the chunk we compressed. + */ + if (mayhem__corrupt_loose) { + logmayhem("corrupt_loose"); + *in0 = *in0 ^ 0xff; + } + + ret = git_deflate(&stream, Z_FINISH); + + if (mayhem__corrupt_loose) + *in0 = *in0 ^ 0xff; + + git_hash_update(&c, in0, stream.next_in - in0); + + /* [5] */ + wr = send_chunk(fd, compressed, stream.next_out - compressed); + if (wr & WR_STOP_THE_MUSIC) { + free(content); + return wr; + } + + stream.next_out = compressed; + stream.avail_out = sizeof(compressed); + + } while (ret == Z_OK); + + /* [3] */ + if (ret != Z_STREAM_END) + BUG("unable to deflate object '%s' (%d)", oid_to_hex(oid), ret); + ret = git_deflate_end_gently(&stream); + if (ret != Z_OK) + BUG("deflateEnd on object '%s' failed (%d)", oid_to_hex(oid), ret); + + /* [6] */ + git_hash_final(oid_check.hash, &c); + if (!oideq(oid, &oid_check)) + BUG("send_loose_object[6]: invalid construction '%s' '%s'", + oid_to_hex(oid), oid_to_hex(&oid_check)); + + /* [5] */ + free(content); + return send_final_chunk(fd); +} + +/* + * Per the GVFS Protocol, a single OID should be in the slash-arg: + * + * GET /gvfs/objects/fc3fff3a25559d2d30d1719c4f4a6d9fe7e05170 HTTP/1.1 + * + * Look it up in our repo (loose or packed) and send it to gvfs-helper + * over the socket as a loose object. + */ +static enum worker_result do__gvfs_objects__get(struct req *req) +{ + struct object_id oid; + + if (!req->slash_args.len || + get_oid_hex(req->slash_args.buf, &oid)) { + logerror("invalid OID in GET gvfs/objects: '%s'", + req->slash_args.buf); + return WR_IO_ERROR; + } + + trace2_printf("%s: GET %s", TR2_CAT, oid_to_hex(&oid)); + + return send_loose_object(&oid, 1); +} + +static enum worker_result read_json_post_body( + struct req *req, + struct oidset *oids, + int *nr_oids) +{ + struct object_id oid; + struct string_list_item *item; + char *post_body = NULL; + const char *v; + ssize_t len_expected = 0; + ssize_t len_received; + const char *pkey; + const char *plbracket; + const char *pstart; + const char *pend; + + for_each_string_list_item(item, &req->header_list) { + if (skip_prefix(item->string, "Content-Length: ", &v)) { + char *p; + len_expected = strtol(v, &p, 10); + break; + } + } + if (!len_expected) { + logerror("no content length in POST"); + return WR_IO_ERROR; + } + post_body = xcalloc(1, len_expected + 1); + if (!post_body) { + logerror("could not malloc buffer for POST body"); + return WR_IO_ERROR; + } + len_received = read_in_full(0, post_body, len_expected); + if (len_received != len_expected) { + logerror("short read in POST (expected %d, received %d)", + (int)len_expected, (int)len_received); + return WR_IO_ERROR; + } + + /* + * A very primitive JSON parser for a very fixed and well-known + * message format. Please don't judge me. + * + * We expect: + * + * ..."objectIds":["","",...""]... + * + * We expect compact (non-pretty) JSON, but do allow it. + */ + pkey = strstr(post_body, "\"objectIds\""); + if (!pkey) + goto could_not_parse_json; + plbracket = strchr(pkey, '['); + if (!plbracket) + goto could_not_parse_json; + pstart = plbracket + 1; + + while (1) { + /* Eat leading whitespace before opening DQUOTE */ + while (*pstart && isspace(*pstart)) + pstart++; + if (!*pstart) + goto could_not_parse_json; + pstart++; + + /* find trailing DQUOTE */ + pend = strchr(pstart, '"'); + if (!pend) + goto could_not_parse_json; + + if (get_oid_hex(pstart, &oid)) + goto could_not_parse_json; + if (!oidset_insert(oids, &oid)) + *nr_oids += 1; + trace2_printf("%s: POST %s", TR2_CAT, oid_to_hex(&oid)); + + /* Eat trailing whitespace after trailing DQUOTE */ + pend++; + while (*pend && isspace(*pend)) + pend++; + if (!*pend) + goto could_not_parse_json; + + /* End of list or is there another OID */ + if (*pend == ']') + break; + if (*pend != ',') + goto could_not_parse_json; + + pstart = pend + 1; + } + + /* + * We do not care about the "commitDepth" parameter. + */ + + free(post_body); + return WR_OK; + +could_not_parse_json: + logerror("could not parse JSON in POST body"); + free(post_body); + return WR_IO_ERROR; +} + +/* + * Since this is a test helper, I'm going to be lazy and + * run pack-objects as a background child using pipe_command + * and get the resulting packfile into a buffer. And then + * the caller can pump it to the client over the socket. + * + * This avoids the need to set up a custom loop (like in + * upload-pack) to drive it and/or the use of a bunch of + * tempfiles. + * + * My assumption here is that we're not testing with GBs + * of data.... + */ +static enum worker_result get_packfile_from_oids( + struct oidset *oids, + struct strbuf *buf_packfile) +{ + struct child_process pack_objects = CHILD_PROCESS_INIT; + struct strbuf buf_child_stdin = STRBUF_INIT; + struct strbuf buf_child_stderr = STRBUF_INIT; + struct oidset_iter iter; + struct object_id *oid; + enum worker_result wr; + int result; + + strvec_push(&pack_objects.args, "git"); + strvec_push(&pack_objects.args, "pack-objects"); + strvec_push(&pack_objects.args, "-q"); + strvec_push(&pack_objects.args, "--revs"); + strvec_push(&pack_objects.args, "--delta-base-offset"); + strvec_push(&pack_objects.args, "--window=0"); + strvec_push(&pack_objects.args, "--depth=4095"); + strvec_push(&pack_objects.args, "--compression=1"); + strvec_push(&pack_objects.args, "--stdout"); + + pack_objects.in = -1; + pack_objects.out = -1; + pack_objects.err = -1; + + oidset_iter_init(oids, &iter); + while ((oid = oidset_iter_next(&iter))) + strbuf_addf(&buf_child_stdin, "%s\n", oid_to_hex(oid)); + strbuf_addstr(&buf_child_stdin, "\n"); + + result = pipe_command(&pack_objects, + buf_child_stdin.buf, buf_child_stdin.len, + buf_packfile, 0, + &buf_child_stderr, 0); + if (result) { + logerror("pack-objects failed: %s", buf_child_stderr.buf); + wr = WR_IO_ERROR; + goto done; + } + + wr = WR_OK; + +done: + strbuf_release(&buf_child_stdin); + strbuf_release(&buf_child_stderr); + + return wr; +} + +static enum worker_result send_packfile_from_buffer(const struct strbuf *packfile) +{ + struct strbuf response_header = STRBUF_INIT; + struct strbuf uuid = STRBUF_INIT; + enum worker_result wr; + + strbuf_addstr(&response_header, "HTTP/1.1 200 OK\r\n"); + strbuf_addstr(&response_header, "Cache-Control: private\r\n"); + strbuf_addstr(&response_header, "Content-Type: application/x-git-packfile\r\n"); + strbuf_addf( &response_header, "Content-Length: %d\r\n", (int)packfile->len); + strbuf_addf( &response_header, "Server: test-gvfs-protocol/%s\r\n", git_version_string); + strbuf_addf( &response_header, "Date: %s\r\n", show_date(time(NULL), 0, DATE_MODE(RFC2822))); + gen_fake_uuid(&uuid); + strbuf_addf( &response_header, "X-VSS-E2EID: %s\r\n", uuid.buf); + strbuf_addstr(&response_header, "\r\n"); + + if (write_in_full(1, response_header.buf, response_header.len) < 0) { + logerror("unable to write response header"); + wr = WR_IO_ERROR; + goto done; + } + + if ((string_list_has_string(&mayhem_list, "bad_post_pack_sha") || + (string_list_has_string(&mayhem_list, "bad_post_pack_sha_1") && + mayhem_child == 0)) && + packfile->len) { + char byte = packfile->buf[packfile->len - 1] ^ 0xff; + + logmayhem("bad_post_pack_sha%s", + string_list_has_string(&mayhem_list, + "bad_post_pack_sha_1") ? + "_1" : ""); + if (write_in_full(1, packfile->buf, packfile->len - 1) < 0 || + write_in_full(1, &byte, 1) < 0) { + logerror("unable to write corrupt response body"); + wr = WR_IO_ERROR; + goto done; + } + if (string_list_has_string(&mayhem_list, + "bad_post_pack_sha_1")) { + wr = WR_MAYHEM | WR_HANGUP; + goto done; + } + } else if (write_in_full(1, packfile->buf, packfile->len) < 0) { + logerror("unable to write response content body"); + wr = WR_IO_ERROR; + goto done; + } + + wr = WR_OK; + +done: + strbuf_release(&uuid); + strbuf_release(&response_header); + + return wr; +} + +/* + * The GVFS Protocol POST verb behaves like GET for non-commit objects + * (in that it just returns the requested object), but for commit + * objects POST *also* returns all trees referenced by the commit. + * + * The goal of this test is to confirm that: + * [] `gvfs-helper post` can request and receive a packfile at all. + * [] `gvfs-helper post` can handle getting either a packfile or a + * loose object. + * + * Therefore, I'm not going to blur the issue and support the custom + * semantics for commit objects. + * + * If one of the OIDs is a commit, `git pack-objects` will completely + * walk the trees and blobs for it and we get that for free. This is + * good enough for our testing. + * + * TODO A proper solution would separate the commit objects and do a + * TODO `rev-list --filter=blobs:none` for them (or use the internal + * TODO list-objects API) and a regular enumeration for the non-commit + * TODO objects. And build an new oidset with union of those and then + * TODO call pack-objects on it instead. + * TODO + * TODO But that's too much trouble for now. + * + * For now, we just need to know if the post asks for a single object, + * is it a commit or non-commit. That is sufficient to know whether + * we should send a packfile or loose object. +*/ +static enum worker_result classify_oids_in_post( + struct oidset *oids, int nr_oids, int *need_packfile) +{ + struct oidset_iter iter; + struct object_id *oid; + enum object_type type; + struct object_info oi = OBJECT_INFO_INIT; + unsigned flags = 0; + + if (nr_oids > 1) { + *need_packfile = 1; + return WR_OK; + } + + /* disable missing-object faulting */ + flags |= OBJECT_INFO_FOR_PREFETCH; + flags |= OBJECT_INFO_LOOKUP_REPLACE; + + oi.typep = &type; + + oidset_iter_init(oids, &iter); + while ((oid = oidset_iter_next(&iter))) { + if (!odb_read_object_info_extended(the_repository->objects, oid, &oi, flags) && + type == OBJ_COMMIT) { + *need_packfile = 1; + return WR_OK; + } + } + + *need_packfile = 0; + return WR_OK; +} + +static enum worker_result do__gvfs_objects__post(struct req *req) +{ + struct oidset oids = OIDSET_INIT; + struct strbuf packfile = STRBUF_INIT; + enum worker_result wr; + int nr_oids = 0; + int need_packfile = 0; + + wr = read_json_post_body(req, &oids, &nr_oids); + if (wr & WR_STOP_THE_MUSIC) + goto done; + + wr = classify_oids_in_post(&oids, nr_oids, &need_packfile); + if (wr & WR_STOP_THE_MUSIC) + goto done; + + if (!need_packfile) { + struct oidset_iter iter; + struct object_id *oid; + + oidset_iter_init(&oids, &iter); + oid = oidset_iter_next(&iter); + + wr = send_loose_object(oid, 1); + } else { + wr = get_packfile_from_oids(&oids, &packfile); + if (wr & WR_STOP_THE_MUSIC) + goto done; + + wr = send_packfile_from_buffer(&packfile); + } + +done: + oidset_clear(&oids); + strbuf_release(&packfile); + + return wr; +} + +/* + * bswap.h only defines big endian functions. + * The GVFS Protocol defines fields in little endian. + */ +static inline uint64_t my_get_le64(uint64_t le_val) +{ +#if GIT_BYTE_ORDER == GIT_LITTLE_ENDIAN + return le_val; +#else + return default_bswap64(le_val); +#endif +} + +static inline uint16_t my_get_le16(uint16_t le_val) +{ +#if GIT_BYTE_ORDER == GIT_LITTLE_ENDIAN + return le_val; +#else + return default_bswap16(le_val); +#endif +} + +/* + * GVFS Protocol headers for the multipack format + * All integer values are little-endian on the wire. + * + * Note: technically, the protocol defines the `ph` fields as signed, but + * that makes a mess of the bswap routines and we're not going to overflow + * them for a very long time. + */ + +static unsigned char v1_h[6] = { 'G', 'P', 'R', 'E', ' ', 0x01 }; + +struct ph { + uint64_t timestamp; + uint64_t len_pack; + uint64_t len_idx; +}; + +/* + * Accumulate a list of commits-and-trees packfiles we have in the local ODB. + * The test script should have pre-created a set of "ct-.pack" and .idx + * files for us. We serve these as is and DO NOT try to dynamically create + * new commits/trees packfiles (like the cache-server does). We are only + * testing if/whether gvfs-helper.exe can receive one or more packfiles and + * idx files over the protocol. + */ +struct ct_pack_item { + struct ph ph; + struct strbuf path_pack; + struct strbuf path_idx; +}; + +static void ct_pack_item__free(struct ct_pack_item *item) +{ + if (!item) + return; + strbuf_release(&item->path_pack); + strbuf_release(&item->path_idx); + free(item); +} + +struct ct_pack_data { + struct ct_pack_item **items; + size_t nr, alloc; +}; + +static void ct_pack_data__release(struct ct_pack_data *data) +{ + size_t k; + + if (!data) + return; + + for (k = 0; k < data->nr; k++) + ct_pack_item__free(data->items[k]); + + FREE_AND_NULL(data->items); + data->nr = 0; + data->alloc = 0; +} + +static void cb_ct_pack(const char *full_path, size_t full_path_len UNUSED, + const char *file_path, void *void_data) +{ + struct ct_pack_data *data = void_data; + struct ct_pack_item *item = NULL; + struct stat st; + const char *v; + + /* + * We only want "ct-.pack" files. The test script creates + * cached commits-and-trees packfiles with this prefix to avoid + * confusion with prefetch packfiles received by gvfs-helper. + */ + if (!ends_with(file_path, ".pack")) + return; + if (!skip_prefix(file_path, "ct-", &v)) + return; + + item = (struct ct_pack_item *)xcalloc(1, sizeof(*item)); + strbuf_init(&item->path_pack, 0); + strbuf_addstr(&item->path_pack, full_path); + + strbuf_init(&item->path_idx, 0); + strbuf_addstr(&item->path_idx, full_path); + strbuf_strip_suffix(&item->path_idx, ".pack"); + strbuf_addstr(&item->path_idx, ".idx"); + + item->ph.timestamp = (uint64_t)strtoul(v, NULL, 10); + + lstat(item->path_pack.buf, &st); + item->ph.len_pack = (uint64_t)st.st_size; + + if (string_list_has_string(&mayhem_list, "no_prefetch_idx")) + item->ph.len_idx = maximum_unsigned_value_of_type(uint64_t); + else if (lstat(item->path_idx.buf, &st) < 0) + item->ph.len_idx = maximum_unsigned_value_of_type(uint64_t); + else + item->ph.len_idx = (uint64_t)st.st_size; + + ALLOC_GROW(data->items, data->nr + 1, data->alloc); + data->items[data->nr++] = item; +} + +/* + * Sort by increasing EPOCH time. + */ +static int ct_pack_sort_compare(const void *_a, const void *_b) +{ + const struct ct_pack_item *a = *(const struct ct_pack_item **)_a; + const struct ct_pack_item *b = *(const struct ct_pack_item **)_b; + return (a->ph.timestamp < b->ph.timestamp) ? -1 : (a->ph.timestamp != b->ph.timestamp); +} + +#define MY_MIN(a, b) ((a) < (b) ? (a) : (b)) + +/* + * Like copy.c:copy_fd(), but corrupt part of the trailing SHA (if the + * given mayhem key is defined) as we copy it to the destination file. + * + * We don't know (or care) if the input file is a pack file or idx + * file, just that the final bytes are part of a SHA that we can + * corrupt. + */ +static int copy_fd_with_checksum_mayhem(int ifd, int ofd, + const char *mayhem_key, + ssize_t nr_wrong_bytes) +{ + off_t in_cur, in_len; + ssize_t bytes_to_copy; + ssize_t bytes_remaining_to_copy; + char buffer[8192]; + + if (!mayhem_key || !*mayhem_key || !nr_wrong_bytes || + !string_list_has_string(&mayhem_list, mayhem_key)) + return copy_fd(ifd, ofd); + + in_cur = lseek(ifd, 0, SEEK_CUR); + if (in_cur < 0) + return in_cur; + + in_len = lseek(ifd, 0, SEEK_END); + if (in_len < 0) + return in_len; + + if (lseek(ifd, in_cur, SEEK_SET) < 0) + return -1; + + /* Copy the entire file except for the last few bytes. */ + + bytes_to_copy = (ssize_t)in_len - nr_wrong_bytes; + bytes_remaining_to_copy = bytes_to_copy; + while (bytes_remaining_to_copy) { + ssize_t to_read = MY_MIN((ssize_t)sizeof(buffer), bytes_remaining_to_copy); + ssize_t len = xread(ifd, buffer, to_read); + + if (!len) + return -1; /* error on unexpected EOF */ + if (len < 0) + return -1; + if (write_in_full(ofd, buffer, len) < 0) + return -1; + + bytes_remaining_to_copy -= len; + } + + /* Read the trailing bytes so that we can alter them before copying. */ + + while (nr_wrong_bytes) { + ssize_t to_read = MY_MIN((ssize_t)sizeof(buffer), nr_wrong_bytes); + ssize_t len = xread(ifd, buffer, to_read); + ssize_t k; + + if (!len) + return -1; /* error on unexpected EOF */ + if (len < 0) + return -1; + + for (k = 0; k < len; k++) + buffer[k] ^= 0xff; + + if (write_in_full(ofd, buffer, len) < 0) + return -1; + + nr_wrong_bytes -= len; + } + + return 0; +} + +static enum worker_result send_ct_item(const struct ct_pack_item *item) +{ + struct ph ph_le; + int fd_pack = -1; + int fd_idx = -1; + enum worker_result wr = WR_OK; + + /* send per-packfile header. all fields are little-endian on the wire. */ + ph_le.timestamp = my_get_le64(item->ph.timestamp); + ph_le.len_pack = my_get_le64(item->ph.len_pack); + ph_le.len_idx = my_get_le64(item->ph.len_idx); + + if (write_in_full(1, &ph_le, sizeof(ph_le)) < 0) { + logerror("unable to write ph_le"); + wr = WR_IO_ERROR; + goto done; + } + + trace2_printf("%s: sending prefetch pack '%s'", TR2_CAT, item->path_pack.buf); + + fd_pack = git_open_cloexec(item->path_pack.buf, O_RDONLY); + if (fd_pack == -1 || + copy_fd_with_checksum_mayhem(fd_pack, 1, "bad_prefetch_pack_sha", 4)) { + logerror("could not send packfile"); + wr = WR_IO_ERROR; + goto done; + } + + if (item->ph.len_idx != maximum_unsigned_value_of_type(uint64_t)) { + trace2_printf("%s: sending prefetch idx '%s'", TR2_CAT, item->path_idx.buf); + + fd_idx = git_open_cloexec(item->path_idx.buf, O_RDONLY); + if (fd_idx == -1 || + copy_fd_with_checksum_mayhem(fd_idx, 1, "bad_prefetch_idx_sha", 4)) { + logerror("could not send idx"); + wr = WR_IO_ERROR; + goto done; + } + } + +done: + if (fd_pack != -1) + close(fd_pack); + if (fd_idx != -1) + close(fd_idx); + return wr; +} + +/* + * The GVFS Protocol defines the lastTimeStamp parameter as the value + * of the last prefetch pack that the client has. Therefore, we only + * want to send newer ones. + */ +static int want_ct_pack(const struct ct_pack_item *item, timestamp_t last_timestamp) +{ + return item->ph.timestamp > last_timestamp; +} + +static enum worker_result send_multipack(struct ct_pack_data *data, + timestamp_t last_timestamp) +{ + struct strbuf response_header = STRBUF_INIT; + struct strbuf uuid = STRBUF_INIT; + enum worker_result wr; + size_t content_len = 0; + unsigned short np = 0; + unsigned short np_le; + size_t k; + + /* + * Precompute the content-length so that we don't have to deal with + * chunking it. + */ + content_len += sizeof(v1_h) + sizeof(np); + for (k = 0; k < data->nr; k++) { + struct ct_pack_item *item = data->items[k]; + + if (!want_ct_pack(item, last_timestamp)) + continue; + + np++; + content_len += sizeof(struct ph); + content_len += item->ph.len_pack; + if (item->ph.len_idx != maximum_unsigned_value_of_type(uint64_t)) + content_len += item->ph.len_idx; + } + + strbuf_addstr(&response_header, "HTTP/1.1 200 OK\r\n"); + strbuf_addstr(&response_header, "Cache-Control: private\r\n"); + strbuf_addstr(&response_header, + "Content-Type: application/x-gvfs-timestamped-packfiles-indexes\r\n"); + strbuf_addf( &response_header, "Content-Length: %d\r\n", (int)content_len); + strbuf_addf( &response_header, "Server: test-gvfs-protocol/%s\r\n", git_version_string); + strbuf_addf( &response_header, "Date: %s\r\n", show_date(time(NULL), 0, DATE_MODE(RFC2822))); + gen_fake_uuid(&uuid); + strbuf_addf( &response_header, "X-VSS-E2EID: %s\r\n", uuid.buf); + strbuf_addstr(&response_header, "\r\n"); + + if (write_in_full(1, response_header.buf, response_header.len) < 0) { + logerror("unable to write response header"); + wr = WR_IO_ERROR; + goto done; + } + + /* send protocol version header */ + if (write_in_full(1, v1_h, sizeof(v1_h)) < 0) { + logerror("unabled to write v1_h"); + wr = WR_IO_ERROR; + goto done; + } + + /* send number of packfiles */ + np_le = my_get_le16(np); + if (write_in_full(1, &np_le, sizeof(np_le)) < 0) { + logerror("unable to write np"); + wr = WR_IO_ERROR; + goto done; + } + + for (k = 0; k < data->nr; k++) { + if (!want_ct_pack(data->items[k], last_timestamp)) + continue; + + wr = send_ct_item(data->items[k]); + if (wr != WR_OK) + goto done; + } + + wr = WR_OK; + +done: + strbuf_release(&uuid); + strbuf_release(&response_header); + + return wr; +} + +static enum worker_result do__gvfs_prefetch__get(struct req *req) +{ + struct ct_pack_data data; + timestamp_t last_timestamp = 0; + enum worker_result wr; + + memset(&data, 0, sizeof(data)); + + if (req->quest_args.len) { + const char *key = strstr(req->quest_args.buf, "lastPackTimestamp="); + if (key) { + const char *val; + if (skip_prefix(key, "lastPackTimestamp=", &val)) { + last_timestamp = strtol(val, NULL, 10); + } + } + } + trace2_printf("%s: prefetch/since %"PRItime, TR2_CAT, last_timestamp); + + for_each_file_in_pack_dir(repo_get_object_directory(the_repository), cb_ct_pack, &data); + QSORT(data.items, data.nr, ct_pack_sort_compare); + + wr = send_multipack(&data, last_timestamp); + + ct_pack_data__release(&data); + + return wr; +} + +/* + * Read the HTTP request up to the start of the optional message-body. + * We do this byte-by-byte because we have keep-alive turned on and + * cannot rely on an EOF. + * + * https://tools.ietf.org/html/rfc7230 + * https://github.com/microsoft/VFSForGit/blob/master/Protocol.md + * + * We cannot call die() here because our caller needs to properly + * respond to the client and/or close the socket before this + * child exits so that the client doesn't get a connection reset + * by peer error. + */ +static enum worker_result req__read(struct req *req, int fd) +{ + struct strbuf h = STRBUF_INIT; + int nr_start_line_fields; + const char *uri_target; + const char *http_version; + const char *gvfs; + + /* + * Read line 0 of the request and split it into component parts: + * + * SP SP CRLF + * + */ + if (strbuf_getwholeline_fd(&req->start_line, fd, '\n') == EOF) + return WR_OK | WR_HANGUP; + + if (string_list_has_string(&mayhem_list, "close_read")) { + logmayhem("close_read"); + return WR_MAYHEM | WR_HANGUP; + } + + if (string_list_has_string(&mayhem_list, "close_read_1") && + mayhem_child == 0) { + /* + * Mayhem: fail the first request, but let retries succeed. + */ + logmayhem("close_read_1"); + return WR_MAYHEM | WR_HANGUP; + } + + strbuf_trim_trailing_newline(&req->start_line); + + nr_start_line_fields = string_list_split(&req->start_line_fields, + req->start_line.buf, + " ", -1); + if (nr_start_line_fields != 3) { + logerror("could not parse request start-line '%s'", + req->start_line.buf); + return WR_IO_ERROR; + } + uri_target = req->start_line_fields.items[1].string; + http_version = req->start_line_fields.items[2].string; + + if (strcmp(http_version, "HTTP/1.1")) { + logerror("unsupported version '%s' (expecting HTTP/1.1)", + http_version); + return WR_IO_ERROR; + } + + /* + * Next, extract the GVFS terms from the . The + * GVFS Protocol defines a REST API containing several GVFS + * commands of the form: + * + * []/gvfs/[/] + * []/gvfs/[?] + * + * For example: + * "GET /gvfs/config HTTP/1.1" + * "GET /gvfs/objects/aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd HTTP/1.1" + * "GET /gvfs/prefetch?lastPackTimestamp=123456789 HTTP/1.1" + * + * "GET //gvfs/config HTTP/1.1" + * "GET //gvfs/objects/aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd HTTP/1.1" + * "GET //gvfs/prefetch?lastPackTimestamp=123456789 HTTP/1.1" + * + * "POST //gvfs/objects HTTP/1.1" + * + * For other testing later, we also allow non-gvfs URLs of the form: + * "GET /[?] HTTP/1.1" + * + * We do not attempt to split the query-params within the args. + * The caller can do that if they need to. + */ + gvfs = strstr(uri_target, "/gvfs/"); + if (gvfs) { + strbuf_add(&req->uri_base, uri_target, (gvfs - uri_target)); + strbuf_trim_trailing_dir_sep(&req->uri_base); + + gvfs += 6; /* skip "/gvfs/" */ + strbuf_add(&req->gvfs_api, "gvfs/", 5); + while (*gvfs && *gvfs != '/' && *gvfs != '?') + strbuf_addch(&req->gvfs_api, *gvfs++); + + /* + */ + if (*gvfs == '/') + strbuf_addstr(&req->slash_args, gvfs + 1); + else if (*gvfs == '?') + strbuf_addstr(&req->quest_args, gvfs + 1); + } else { + + const char *quest = strchr(uri_target, '?'); + + if (quest) { + strbuf_add(&req->uri_base, uri_target, (quest - uri_target)); + strbuf_trim_trailing_dir_sep(&req->uri_base); + strbuf_addstr(&req->quest_args, quest + 1); + } else { + strbuf_addstr(&req->uri_base, uri_target); + strbuf_trim_trailing_dir_sep(&req->uri_base); + } + } + + /* + * Read the set of HTTP headers into a string-list. + */ + while (1) { + if (strbuf_getwholeline_fd(&h, fd, '\n') == EOF) + goto done; + strbuf_trim_trailing_newline(&h); + + if (!h.len) + goto done; /* a blank line ends the header */ + + string_list_append(&req->header_list, h.buf); + } + + /* + * TODO If the set of HTTP headers includes things like: + * TODO + * TODO Connection: Upgrade, HTTP2-Settings + * TODO Upgrade: h2c + * TODO HTTP2-Settings: AAMAAABkAARAAAAAAAIAAAAA + * TODO + * TODO then the client is asking to optionally switch to HTTP/2. + * TODO + * TODO We currently DO NOT support that (and I don't currently + * TODO see a need to do so (because we don't need the multiplexed + * TODO streams feature (because the client never asks for n packfiles + * TODO at the same time))). + * TODO + * TODO https://en.wikipedia.org/wiki/HTTP/1.1_Upgrade_header + */ + + /* + * We do not attempt to read the , if it exists. + * We let our caller read/chunk it in as appropriate. + */ +done: + + /* + * Log selected test headers if present. + */ + { + struct string_list_item *item; + for_each_string_list_item(item, &req->header_list) { + if (starts_with(item->string, "X-Session-Id:") || + starts_with(item->string, "X-Test-Header:")) + loginfo("Received header: %s", item->string); + } + } + +#if 0 + /* + * This is useful for debugging the request, but very noisy. + */ + if (trace2_is_enabled()) { + struct string_list_item *item; + trace2_printf("%s: %s", TR2_CAT, req->start_line.buf); + for_each_string_list_item(item, &req->start_line_fields) + trace2_printf("%s: Field: %s", TR2_CAT, item->string); + trace2_printf("%s: [uri-base '%s'][gvfs '%s'][args '%s' '%s']", + TR2_CAT, + req->uri_base.buf, + req->gvfs_api.buf, + req->slash_args.buf, + req->quest_args.buf); + for_each_string_list_item(item, &req->header_list) + trace2_printf("%s: Hdrs: %s", TR2_CAT, item->string); + } +#endif + + strbuf_release(&h); + + return WR_OK; +} + +static enum worker_result dispatch(struct req *req) +{ + static regex_t *smart_http_regex; + static int initialized; + const char *method; + enum worker_result wr; + + if (strstr(req->uri_base.buf, MY_SERVER_TYPE__CACHE)) { + if (string_list_has_string(&mayhem_list, "cache_http_404")) { + logmayhem("cache_http_404"); + return send_http_error(1, 404, "Not Found", -1, + WR_MAYHEM | WR_HANGUP); + } + if (string_list_has_string(&mayhem_list, "cache_http_503")) { + logmayhem("cache_http_503"); + return send_http_error(1, 503, "Service Unavailable", 2, + WR_MAYHEM | WR_HANGUP); + } + } + + if (string_list_has_string(&mayhem_list, "close_no_write")) { + logmayhem("close_no_write"); + return WR_MAYHEM | WR_HANGUP; + } + if (string_list_has_string(&mayhem_list, "http_503")) { + logmayhem("http_503"); + return send_http_error(1, 503, "Service Unavailable", 2, + WR_MAYHEM | WR_HANGUP); + } + if (string_list_has_string(&mayhem_list, "http_429")) { + logmayhem("http_429"); + return send_http_error(1, 429, "Too Many Requests", 2, + WR_MAYHEM | WR_HANGUP); + } + if (string_list_has_string(&mayhem_list, "http_429_1") && + mayhem_child == 0) { + logmayhem("http_429_1"); + return send_http_error(1, 429, "Too Many Requests", 2, + WR_MAYHEM | WR_HANGUP); + } + if (mayhem_try_auth(req, &wr)) + return wr; + + method = req->start_line_fields.items[0].string; + + if (!strcmp(req->gvfs_api.buf, "gvfs/objects")) { + + if (!strcmp(method, "GET")) + return do__gvfs_objects__get(req); + if (!strcmp(method, "POST")) + return do__gvfs_objects__post(req); + } + + if (!strcmp(req->gvfs_api.buf, "gvfs/config")) { + + if (!strcmp(method, "GET")) + return do__gvfs_config__get(req); + } + + if (!strcmp(req->gvfs_api.buf, "gvfs/prefetch")) { + + if (!strcmp(method, "GET")) + return do__gvfs_prefetch__get(req); + } + + if (!initialized) { + smart_http_regex = xmalloc(sizeof(*smart_http_regex)); + if (regcomp(smart_http_regex, "^/(HEAD|info/refs|" + "objects/info/[^/]+|git-(upload|receive)-pack)$", + REG_EXTENDED)) { + warning("could not compile smart HTTP regex"); + smart_http_regex = NULL; + } + initialized = 1; + } + + if (smart_http_regex && + !regexec(smart_http_regex, req->uri_base.buf, 0, NULL, 0)) { + const char *ok = "HTTP/1.1 200 OK\r\n"; + struct child_process cp = CHILD_PROCESS_INIT; + size_t i; + int res; + + if (write(1, ok, strlen(ok)) < 0) + return error(_("could not send '%s'"), ok); + + strvec_pushf(&cp.env, "REQUEST_METHOD=%s", method); + strvec_pushf(&cp.env, "PATH_TRANSLATED=%s", + req->uri_base.buf); + /* Prevent MSYS2 from "converting to a Windows path" */ + strvec_pushf(&cp.env, + "MSYS2_ENV_CONV_EXCL=PATH_TRANSLATED"); + strvec_push(&cp.env, "SERVER_PROTOCOL=HTTP/1.1"); + if (req->quest_args.len) + strvec_pushf(&cp.env, "QUERY_STRING=%s", + req->quest_args.buf); + for (i = 0; i < req->header_list.nr; i++) { + const char *header = req->header_list.items[i].string; + if (!strncasecmp("Content-Type: ", header, 14)) + strvec_pushf(&cp.env, "CONTENT_TYPE=%s", + header + 14); + else if (!strncasecmp("Content-Length: ", header, 16)) + strvec_pushf(&cp.env, "CONTENT_LENGTH=%s", + header + 16); + } + cp.git_cmd = 1; + strvec_push(&cp.args, "http-backend"); + res = run_command(&cp); + close(1); + close(0); + return !!res; + } + + return send_http_error(1, 501, "Not Implemented", -1, + WR_OK | WR_HANGUP); +} + +static enum worker_result worker(void) +{ + struct req req = REQ__INIT; + char *client_addr = getenv("REMOTE_ADDR"); + char *client_port = getenv("REMOTE_PORT"); + enum worker_result wr = WR_OK; + + if (client_addr) + loginfo("Connection from %s:%s", client_addr, client_port); + + set_keep_alive(0); + + while (1) { + req__release(&req); + + alarm(init_timeout ? init_timeout : timeout); + wr = req__read(&req, 0); + alarm(0); + + if (wr & WR_STOP_THE_MUSIC) + break; + + wr = dispatch(&req); + if (wr & WR_STOP_THE_MUSIC) + break; + } + + close(0); + close(1); + + req__release(&req); + return !!(wr & WR_IO_ERROR); +} + +////////////////////////////////////////////////////////////////// +// This section contains the listener and child-process management +// code used by the primary instance to accept incoming connections +// and dispatch them to async child process "worker" instances. +////////////////////////////////////////////////////////////////// + +static int addrcmp(const struct sockaddr_storage *s1, + const struct sockaddr_storage *s2) +{ + const struct sockaddr *sa1 = (const struct sockaddr*) s1; + const struct sockaddr *sa2 = (const struct sockaddr*) s2; + + if (sa1->sa_family != sa2->sa_family) + return sa1->sa_family - sa2->sa_family; + if (sa1->sa_family == AF_INET) + return memcmp(&((struct sockaddr_in *)s1)->sin_addr, + &((struct sockaddr_in *)s2)->sin_addr, + sizeof(struct in_addr)); +#ifndef NO_IPV6 + if (sa1->sa_family == AF_INET6) + return memcmp(&((struct sockaddr_in6 *)s1)->sin6_addr, + &((struct sockaddr_in6 *)s2)->sin6_addr, + sizeof(struct in6_addr)); +#endif + return 0; +} + +static int max_connections = 32; + +static unsigned int live_children; + +static struct child { + struct child *next; + struct child_process cld; + struct sockaddr_storage address; +} *firstborn; + +static void add_child(struct child_process *cld, struct sockaddr *addr, socklen_t addrlen) +{ + struct child *newborn, **cradle; + + newborn = xcalloc(1, sizeof(*newborn)); + live_children++; + memcpy(&newborn->cld, cld, sizeof(*cld)); + memcpy(&newborn->address, addr, addrlen); + for (cradle = &firstborn; *cradle; cradle = &(*cradle)->next) + if (!addrcmp(&(*cradle)->address, &newborn->address)) + break; + newborn->next = *cradle; + *cradle = newborn; +} + +/* + * This gets called if the number of connections grows + * past "max_connections". + * + * We kill the newest connection from a duplicate IP. + */ +static void kill_some_child(void) +{ + const struct child *blanket, *next; + + if (!(blanket = firstborn)) + return; + + for (; (next = blanket->next); blanket = next) + if (!addrcmp(&blanket->address, &next->address)) { + kill(blanket->cld.pid, SIGTERM); + break; + } +} + +static void check_dead_children(void) +{ + int status; + pid_t pid; + + struct child **cradle, *blanket; + for (cradle = &firstborn; (blanket = *cradle);) + if ((pid = waitpid(blanket->cld.pid, &status, WNOHANG)) > 1) { + const char *dead = ""; + if (status) + dead = " (with error)"; + loginfo("[%"PRIuMAX"] Disconnected%s", (uintmax_t)pid, dead); + + /* remove the child */ + *cradle = blanket->next; + live_children--; + child_process_clear(&blanket->cld); + free(blanket); + } else + cradle = &blanket->next; +} + +static struct strvec cld_argv = STRVEC_INIT; +static void handle(int incoming, struct sockaddr *addr, socklen_t addrlen) +{ + struct child_process cld = CHILD_PROCESS_INIT; + + if (max_connections >= 0 && live_children >= (unsigned int)max_connections) { + kill_some_child(); + sleep(1); /* give it some time to die */ + check_dead_children(); + if (live_children >= (unsigned int)max_connections) { + close(incoming); + logerror("Too many children, dropping connection"); + return; + } + } + + if (addr->sa_family == AF_INET) { + char buf[128] = ""; + struct sockaddr_in *sin_addr = (void *) addr; + inet_ntop(addr->sa_family, &sin_addr->sin_addr, buf, sizeof(buf)); + strvec_pushf(&cld.env, "REMOTE_ADDR=%s", buf); + strvec_pushf(&cld.env, "REMOTE_PORT=%d", + ntohs(sin_addr->sin_port)); +#ifndef NO_IPV6 + } else if (addr->sa_family == AF_INET6) { + char buf[128] = ""; + struct sockaddr_in6 *sin6_addr = (void *) addr; + inet_ntop(AF_INET6, &sin6_addr->sin6_addr, buf, sizeof(buf)); + strvec_pushf(&cld.env, "REMOTE_ADDR=[%s]", buf); + strvec_pushf(&cld.env, "REMOTE_PORT=%d", + ntohs(sin6_addr->sin6_port)); +#endif + } + + if (mayhem_list.nr) { + strvec_pushf(&cld.env, "MAYHEM_CHILD=%d", + mayhem_child++); + } + + strvec_pushv(&cld.args, cld_argv.v); + cld.in = incoming; + cld.out = dup(incoming); + + if (cld.out < 0) + logerror("could not dup() `incoming`"); + else if (start_command(&cld)) + logerror("unable to fork"); + else + add_child(&cld, addr, addrlen); +} + +static void child_handler(int signo UNUSED) +{ + /* + * Otherwise empty handler because systemcalls will get interrupted + * upon signal receipt + * SysV needs the handler to be rearmed + */ + signal(SIGCHLD, child_handler); +} + +static int set_reuse_addr(int sockfd) +{ + int on = 1; + + if (!reuseaddr) + return 0; + return setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, + &on, sizeof(on)); +} + +struct socketlist { + int *list; + size_t nr; + size_t alloc; +}; + +static const char *ip2str(int family, struct sockaddr *sin, socklen_t len) +{ +#ifdef NO_IPV6 + static char ip[INET_ADDRSTRLEN]; +#else + static char ip[INET6_ADDRSTRLEN]; +#endif + + switch (family) { +#ifndef NO_IPV6 + case AF_INET6: + inet_ntop(family, &((struct sockaddr_in6*)sin)->sin6_addr, ip, len); + break; +#endif + case AF_INET: + inet_ntop(family, &((struct sockaddr_in*)sin)->sin_addr, ip, len); + break; + default: + xsnprintf(ip, sizeof(ip), ""); + } + return ip; +} + +#ifndef NO_IPV6 + +static int setup_named_sock(const char *listen_addr, int listen_port, struct socketlist *socklist) +{ + int socknum = 0; + char pbuf[NI_MAXSERV]; + struct addrinfo hints, *ai0, *ai; + int gai; + long flags; + + xsnprintf(pbuf, sizeof(pbuf), "%d", listen_port); + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + hints.ai_flags = AI_PASSIVE; + + gai = getaddrinfo(listen_addr, pbuf, &hints, &ai0); + if (gai) { + logerror("getaddrinfo() for %s failed: %s", listen_addr, gai_strerror(gai)); + return 0; + } + + for (ai = ai0; ai; ai = ai->ai_next) { + int sockfd; + + sockfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (sockfd < 0) + continue; + if (sockfd >= FD_SETSIZE) { + logerror("Socket descriptor too large"); + close(sockfd); + continue; + } + +#ifdef IPV6_V6ONLY + if (ai->ai_family == AF_INET6) { + int on = 1; + setsockopt(sockfd, IPPROTO_IPV6, IPV6_V6ONLY, + &on, sizeof(on)); + /* Note: error is not fatal */ + } +#endif + + if (set_reuse_addr(sockfd)) { + logerror("Could not set SO_REUSEADDR: %s", strerror(errno)); + close(sockfd); + continue; + } + + set_keep_alive(sockfd); + + if (bind(sockfd, ai->ai_addr, ai->ai_addrlen) < 0) { + logerror("Could not bind to %s: %s", + ip2str(ai->ai_family, ai->ai_addr, ai->ai_addrlen), + strerror(errno)); + close(sockfd); + continue; /* not fatal */ + } + if (listen(sockfd, 5) < 0) { + logerror("Could not listen to %s: %s", + ip2str(ai->ai_family, ai->ai_addr, ai->ai_addrlen), + strerror(errno)); + close(sockfd); + continue; /* not fatal */ + } + + flags = fcntl(sockfd, F_GETFD, 0); + if (flags >= 0) + fcntl(sockfd, F_SETFD, flags | FD_CLOEXEC); + + ALLOC_GROW(socklist->list, socklist->nr + 1, socklist->alloc); + socklist->list[socklist->nr++] = sockfd; + socknum++; + } + + freeaddrinfo(ai0); + + return socknum; +} + +#else /* NO_IPV6 */ + +static int setup_named_sock(char *listen_addr, int listen_port, struct socketlist *socklist) +{ + struct sockaddr_in sin; + int sockfd; + long flags; + + memset(&sin, 0, sizeof sin); + sin.sin_family = AF_INET; + sin.sin_port = htons(listen_port); + + if (listen_addr) { + /* Well, host better be an IP address here. */ + if (inet_pton(AF_INET, listen_addr, &sin.sin_addr.s_addr) <= 0) + return 0; + } else { + sin.sin_addr.s_addr = htonl(INADDR_ANY); + } + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) + return 0; + + if (set_reuse_addr(sockfd)) { + logerror("Could not set SO_REUSEADDR: %s", strerror(errno)); + close(sockfd); + return 0; + } + + set_keep_alive(sockfd); + + if ( bind(sockfd, (struct sockaddr *)&sin, sizeof sin) < 0 ) { + logerror("Could not bind to %s: %s", + ip2str(AF_INET, (struct sockaddr *)&sin, sizeof(sin)), + strerror(errno)); + close(sockfd); + return 0; + } + + if (listen(sockfd, 5) < 0) { + logerror("Could not listen to %s: %s", + ip2str(AF_INET, (struct sockaddr *)&sin, sizeof(sin)), + strerror(errno)); + close(sockfd); + return 0; + } + + flags = fcntl(sockfd, F_GETFD, 0); + if (flags >= 0) + fcntl(sockfd, F_SETFD, flags | FD_CLOEXEC); + + ALLOC_GROW(socklist->list, socklist->nr + 1, socklist->alloc); + socklist->list[socklist->nr++] = sockfd; + return 1; +} + +#endif + +static void socksetup(struct string_list *listen_addr, int listen_port, struct socketlist *socklist) +{ + if (!listen_addr->nr) + setup_named_sock("127.0.0.1", listen_port, socklist); + else { + size_t i; + int socknum; + for (i = 0; i < listen_addr->nr; i++) { + socknum = setup_named_sock(listen_addr->items[i].string, + listen_port, socklist); + + if (socknum == 0) + logerror("unable to allocate any listen sockets for host %s on port %u", + listen_addr->items[i].string, listen_port); + } + } +} + +static int service_loop(struct socketlist *socklist) +{ + struct pollfd *pfd; + size_t i; + + CALLOC_ARRAY(pfd, socklist->nr); + + for (i = 0; i < socklist->nr; i++) { + pfd[i].fd = socklist->list[i]; + pfd[i].events = POLLIN; + } + + signal(SIGCHLD, child_handler); + + for (;;) { + size_t i; + int nr_ready; + int timeout = (pid_file ? 100 : -1); + + check_dead_children(); + + nr_ready = poll(pfd, socklist->nr, timeout); + if (nr_ready < 0) { + if (errno != EINTR) { + logerror("Poll failed, resuming: %s", + strerror(errno)); + sleep(1); + } + continue; + } + else if (nr_ready == 0) { + /* + * If we have a pid_file, then we watch it. + * If someone deletes it, we shutdown the service. + * The shell scripts in the test suite will use this. + */ + if (!pid_file || file_exists(pid_file)) + continue; + goto shutdown; + } + + for (i = 0; i < socklist->nr; i++) { + if (pfd[i].revents & POLLIN) { + union { + struct sockaddr sa; + struct sockaddr_in sai; +#ifndef NO_IPV6 + struct sockaddr_in6 sai6; +#endif + } ss; + socklen_t sslen = sizeof(ss); + int incoming = accept(pfd[i].fd, &ss.sa, &sslen); + if (incoming < 0) { + switch (errno) { + case EAGAIN: + case EINTR: + case ECONNABORTED: + continue; + default: + die_errno("accept returned"); + } + } + handle(incoming, &ss.sa, sslen); + } + } + } + +shutdown: + loginfo("Starting graceful shutdown (pid-file gone)"); + for (i = 0; i < socklist->nr; i++) + close(socklist->list[i]); + free(socklist->list); + free(pfd); + + return 0; +} + +static int serve(struct string_list *listen_addr, int listen_port) +{ + struct socketlist socklist = { NULL, 0, 0 }; + + socksetup(listen_addr, listen_port, &socklist); + if (socklist.nr == 0) + die("unable to allocate any listen sockets on port %u", + listen_port); + + loginfo("Ready to rumble"); + + /* + * Wait to create the pid-file until we've setup the sockets + * and are open for business. + */ + if (pid_file) + write_file(pid_file, "%"PRIuMAX, (uintmax_t) getpid()); + + return service_loop(&socklist); +} + +////////////////////////////////////////////////////////////////// +// This section is executed by both the primary instance and all +// worker instances. So, yes, each child-process re-parses the +// command line argument and re-discovers how it should behave. +////////////////////////////////////////////////////////////////// + +int cmd_main(int argc, const char **argv) +{ + int listen_port = 0; + static struct string_list listen_addr = STRING_LIST_INIT_NODUP; + int worker_mode = 0; + int i; + + trace2_cmd_name("test-gvfs-protocol"); + setup_git_directory_gently(the_repository, NULL); + + for (i = 1; i < argc; i++) { + const char *arg = argv[i]; + const char *v; + + if (skip_prefix(arg, "--listen=", &v)) { + string_list_append_nodup(&listen_addr, xstrdup_tolower(v)); + continue; + } + if (skip_prefix(arg, "--port=", &v)) { + char *end; + unsigned long n; + n = strtoul(v, &end, 0); + if (*v && !*end) { + listen_port = n; + continue; + } + } + if (!strcmp(arg, "--worker")) { + worker_mode = 1; + trace2_cmd_mode("worker"); + continue; + } + if (!strcmp(arg, "--verbose")) { + verbose = 1; + continue; + } + if (skip_prefix(arg, "--timeout=", &v)) { + timeout = atoi(v); + continue; + } + if (skip_prefix(arg, "--init-timeout=", &v)) { + init_timeout = atoi(v); + continue; + } + if (skip_prefix(arg, "--max-connections=", &v)) { + max_connections = atoi(v); + if (max_connections < 0) + max_connections = 0; /* unlimited */ + continue; + } + if (!strcmp(arg, "--reuseaddr")) { + reuseaddr = 1; + continue; + } + if (skip_prefix(arg, "--pid-file=", &v)) { + pid_file = v; + continue; + } + if (skip_prefix(arg, "--mayhem=", &v)) { + string_list_append(&mayhem_list, v); + continue; + } + + usage(test_gvfs_protocol_usage); + } + + /* avoid splitting a message in the middle */ + setvbuf(stderr, NULL, _IOFBF, 4096); + + if (listen_port == 0) + listen_port = DEFAULT_GIT_PORT; + + /* + * If no --listen= args are given, the setup_named_sock() + * code will use receive a NULL address and set INADDR_ANY. + * This exposes both internal and external interfaces on the + * port. + * + * Disallow that and default to the internal-use-only loopback + * address. + */ + if (!listen_addr.nr) + string_list_append(&listen_addr, "127.0.0.1"); + + /* + * worker_mode is set in our own child process instances + * (that are bound to a connected socket from a client). + */ + if (worker_mode) { + if (mayhem_list.nr) { + const char *string = getenv("MAYHEM_CHILD"); + if (string && *string) + mayhem_child = atoi(string); + } + + build_gvfs_config_json(&jw_config, &listen_addr, listen_port); + + return worker(); + } + + /* + * `cld_argv` is a bit of a clever hack. The top-level instance + * of test-gvfs-protocol.exe does the normal bind/listen/accept + * stuff. For each incoming socket, the top-level process spawns + * a child instance of test-gvfs-protocol.exe *WITH* the additional + * `--worker` argument. This causes the child to set `worker_mode` + * and immediately call `worker()` using the connected socket (and + * without the usual need for fork() or threads). + * + * The magic here is made possible because `cld_argv` is static + * and handle() (called by service_loop()) knows about it. + */ + strvec_push(&cld_argv, argv[0]); + strvec_push(&cld_argv, "--worker"); + for (i = 1; i < argc; ++i) + strvec_push(&cld_argv, argv[i]); + + /* + * Setup primary instance to listen for connections. + */ + return serve(&listen_addr, listen_port); +} diff --git a/t/helper/test-repository.c b/t/helper/test-repository.c index 9ba94cdffa4c47..b999d71b5b7f65 100644 --- a/t/helper/test-repository.c +++ b/t/helper/test-repository.c @@ -23,6 +23,8 @@ static void test_parse_commit_in_graph(const char *gitdir, const char *worktree, repo_set_hash_algo(the_repository, hash_algo_by_ptr(r.hash_algo)); c = lookup_commit(&r, commit_oid); + if (!c) + die("Could not look up %s", oid_to_hex(commit_oid)); if (!parse_commit_in_graph(&r, c)) die("Couldn't parse commit"); diff --git a/t/helper/test-rot13-filter.c b/t/helper/test-rot13-filter.c index ad37e1003445aa..b874992196e91b 100644 --- a/t/helper/test-rot13-filter.c +++ b/t/helper/test-rot13-filter.c @@ -215,7 +215,7 @@ static void command_loop(void) /* Read until flush */ while ((buf = packet_read_line(0, NULL))) { - if (!strcmp(buf, "can-delay=1")) { + if (!strcmp(buf, "can-delay=1")) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand entry = strmap_get(&delay, pathname); if (entry && !entry->requested) entry->requested = 1; @@ -308,11 +308,11 @@ static void packet_initialize(void) { char *pkt_buf = packet_read_line(0, NULL); - if (!pkt_buf || strcmp(pkt_buf, "git-filter-client")) + if (!pkt_buf || strcmp(pkt_buf, "git-filter-client")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die("bad initialize: '%s'", str_or_null(pkt_buf)); pkt_buf = packet_read_line(0, NULL); - if (!pkt_buf || strcmp(pkt_buf, "version=2")) + if (!pkt_buf || strcmp(pkt_buf, "version=2")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die("bad version: '%s'", str_or_null(pkt_buf)); pkt_buf = packet_read_line(0, NULL); diff --git a/t/lib-gvfs-helper.sh b/t/lib-gvfs-helper.sh new file mode 100644 index 00000000000000..a85959c489bb9f --- /dev/null +++ b/t/lib-gvfs-helper.sh @@ -0,0 +1,553 @@ +# Shared library for gvfs-helper tests +# +# This file is sourced by t579*-gvfs-helper*.sh scripts. +# The sourcing script MUST call: +# 1. . ./test-lib.sh +# 2. . "$TEST_DIRECTORY"/lib-gvfs-helper.sh +# 3. init_gvfs_helper_vars +# 4. test_expect_success 'setup repos' 'setup_gvfs_repos' + +# Set the port for t/helper/test-gvfs-protocol.exe from either the +# environment or from the test number of this shell script. +# +test_set_port GIT_TEST_GVFS_PROTOCOL_PORT + +# Setup the following repos: +# +# repo_src: +# A normal, no-magic, fully-populated clone of something. +# No GVFS (aka VFS4G). No Scalar. No partial-clone. +# This will be used by "t/helper/test-gvfs-protocol.exe" +# to serve objects. +# +# repo_t1: +# An empty repo with no contents nor commits. That is, +# everything is missing. For the tests based on this repo, +# we don't care why it is missing objects (or if we could +# actually use it). We are only testing explicit object +# fetching using gvfs-helper.exe in isolation. +# +# repo_t2: +# Another empty repo to use after we contaminate t1. +# +REPO_SRC="$(pwd)"/repo_src +REPO_T1="$(pwd)"/repo_t1 +REPO_T2="$(pwd)"/repo_t2 + +# Setup some loopback URLs where test-gvfs-protocol.exe will be +# listening. We will spawn it directly inside the repo_src directory, +# so we don't need any of the directory mapping or configuration +# machinery found in "git-daemon.exe" or "git-http-backend.exe". +# +# This lets us use the "uri-base" part of the URL (prior to the REST +# API "/gvfs/") to control how our mock server responds. For +# example, only the origin (main Git) server supports "/gvfs/config". +# +# For example, this means that if we add a remote containing $ORIGIN_URL, +# it will work with gvfs-helper, but not for fetch (without some mapping +# tricks). +# +HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_PORT +ORIGIN_URL=http://$HOST_PORT/servertype/origin +CACHE_URL=http://$HOST_PORT/servertype/cache + +SHARED_CACHE_T1="$(pwd)"/shared_cache_t1 +SHARED_CACHE_T2="$(pwd)"/shared_cache_t2 + +# The pid-file is created by test-gvfs-protocol.exe when it starts. +# The server will shut down if/when we delete it. (This is a little +# easier than killing it by PID.) +# +PID_FILE="$(pwd)"/pid-file-gvfs.pid +SERVER_LOG="$(pwd)"/OUT.gvfs.server.log + +# Helper functions to compute port, pid-file, and log for a given +# port increment. An increment of 0 (or empty) uses the base values. +# +# Ensure we don't overlap with any other test port by modifying a +# significant bit. +server_port () { + local instance="${1:-0}" + echo $(($GIT_TEST_GVFS_PROTOCOL_PORT + 10000 * $instance)) +} + +server_pid_file () { + local instance="${1:-0}" + if test "$instance" -eq 0 + then + echo "$PID_FILE" + else + echo "$(pwd)/pid-file-$instance.pid" + fi +} + +server_log_file () { + local instance="${1:-0}" + if test "$instance" -eq 0 + then + echo "$SERVER_LOG" + else + echo "$(pwd)/OUT.server-$instance.log" + fi +} + +# Helper to build a cache-server URL for a given port increment. +# +cache_server_url () { + local instance="${1:-0}" + local port="$(server_port "$instance")" + echo "http://127.0.0.1:$port/servertype/cache" +} + +PATH="$GIT_BUILD_DIR/t/helper/:$PATH" && export PATH + +OIDS_FILE="$(pwd)"/oid_list.txt +OIDS_CT_FILE="$(pwd)"/oid_ct_list.txt +OIDS_BLOBS_FILE="$(pwd)"/oids_blobs_file.txt +OID_ONE_BLOB_FILE="$(pwd)"/oid_one_blob_file.txt +OID_ONE_COMMIT_FILE="$(pwd)"/oid_one_commit_file.txt + +# Get a list of available OIDs in repo_src so that we can try to fetch +# them and so that we don't have to hard-code a list of known OIDs. +# This doesn't need to be a complete list -- just enough to drive some +# representative tests. +# +# Optionally require that we find a minimum number of OIDs. +# +get_list_of_oids () { + git -C "$REPO_SRC" rev-list --objects HEAD | sed 's/ .*//' | sort >"$OIDS_FILE" + + if test $# -eq 1 + then + actual_nr=$(wc -l <"$OIDS_FILE") + if test $actual_nr -lt $1 + then + echo "get_list_of_oids: insufficient data. Need $1 OIDs." + return 1 + fi + fi + return 0 +} + +get_list_of_blobs_oids () { + git -C "$REPO_SRC" ls-tree HEAD | grep ' blob ' | awk "{print \$3}" | sort >"$OIDS_BLOBS_FILE" + head -1 <"$OIDS_BLOBS_FILE" >"$OID_ONE_BLOB_FILE" +} + +get_list_of_commit_and_tree_oids () { + git -C "$REPO_SRC" cat-file --batch-check --batch-all-objects | awk "/commit|tree/ {print \$1}" | sort >"$OIDS_CT_FILE" + + if test $# -eq 1 + then + actual_nr=$(wc -l <"$OIDS_CT_FILE") + if test $actual_nr -lt $1 + then + echo "get_list_of_commit_and_tree_oids: insufficient data. Need $1 OIDs." + return 1 + fi + fi + return 0 +} + +get_one_commit_oid () { + git -C "$REPO_SRC" rev-parse HEAD >"$OID_ONE_COMMIT_FILE" + return 0 +} + +# Create a commits-and-trees packfile for use with "prefetch" +# using the given range of commits. +# +create_commits_and_trees_packfile () { + if test $# -eq 2 + then + epoch=$1 + revs=$2 + else + echo "create_commits_and_trees_packfile: Need 2 args" + return 1 + fi + + pack_file="$REPO_SRC"/.git/objects/pack/ct-$epoch.pack + idx_file="$REPO_SRC"/.git/objects/pack/ct-$epoch.idx + + git -C "$REPO_SRC" pack-objects --stdout --revs --filter=blob:none \ + >"$pack_file" <<-EOF + $revs + EOF + git -C "$REPO_SRC" index-pack -o "$idx_file" "$pack_file" + return 0 +} + +test_expect_success 'setup repos' ' + test_create_repo "$REPO_SRC" && + git -C "$REPO_SRC" branch -M main && + # + # test_commit_bulk() does magic to create a packfile containing + # the new commits. + # + # We create branches in repo_src, but also remember the branch OIDs + # in files so that we can refer to them in repo_t1, which will not + # have the commits locally (because we do not clone or fetch). + # + test_commit_bulk -C "$REPO_SRC" --filename="batch_a.%s.t" 9 && + git -C "$REPO_SRC" branch B1 && + git -C "$REPO_SRC" rev-parse refs/heads/main >m1.branch && + # + test_commit_bulk -C "$REPO_SRC" --filename="batch_b.%s.t" 9 && + git -C "$REPO_SRC" branch B2 && + git -C "$REPO_SRC" rev-parse refs/heads/main >m2.branch && + # + # test_commit() creates commits, trees, tags, and blobs and leave + # them loose. + # + test_config gc.auto 0 && + # + test_commit -C "$REPO_SRC" file1.txt && + test_commit -C "$REPO_SRC" file2.txt && + test_commit -C "$REPO_SRC" file3.txt && + test_commit -C "$REPO_SRC" file4.txt && + test_commit -C "$REPO_SRC" file5.txt && + test_commit -C "$REPO_SRC" file6.txt && + test_commit -C "$REPO_SRC" file7.txt && + test_commit -C "$REPO_SRC" file8.txt && + test_commit -C "$REPO_SRC" file9.txt && + git -C "$REPO_SRC" branch B3 && + git -C "$REPO_SRC" rev-parse refs/heads/main >m3.branch && + # + # Create some commits-and-trees-only packfiles for testing prefetch. + # Set arbitrary EPOCH times to make it easier to test fetch-since. + # + create_commits_and_trees_packfile 1000000000 B1 && + create_commits_and_trees_packfile 1100000000 B1..B2 && + create_commits_and_trees_packfile 1200000000 B2..B3 && + # + # gvfs-helper.exe writes downloaded objects to a shared-cache directory + # rather than the ODB inside the .git directory. + # + mkdir "$SHARED_CACHE_T1" && + mkdir "$SHARED_CACHE_T1/pack" && + mkdir "$SHARED_CACHE_T1/info" && + # + mkdir "$SHARED_CACHE_T2" && + mkdir "$SHARED_CACHE_T2/pack" && + mkdir "$SHARED_CACHE_T2/info" && + # + # setup repo_t1 and point all of the gvfs.* values to repo_src. + # + test_create_repo "$REPO_T1" && + git -C "$REPO_T1" branch -M main && + git -C "$REPO_T1" remote add origin $ORIGIN_URL && + git -C "$REPO_T1" config --local gvfs.cache-server $CACHE_URL && + git -C "$REPO_T1" config --local gvfs.sharedCache "$SHARED_CACHE_T1" && + echo "$SHARED_CACHE_T1" >> "$REPO_T1"/.git/objects/info/alternates && + # + test_create_repo "$REPO_T2" && + git -C "$REPO_T2" branch -M main && + git -C "$REPO_T2" remote add origin $ORIGIN_URL && + git -C "$REPO_T2" config --local gvfs.cache-server $CACHE_URL && + git -C "$REPO_T2" config --local gvfs.sharedCache "$SHARED_CACHE_T2" && + echo "$SHARED_CACHE_T2" >> "$REPO_T2"/.git/objects/info/alternates && + # + # + # + cat <<-EOF >creds.txt && + username=x + password=y + EOF + cat <<-EOF >creds.sh && + #!/bin/sh + cat "$(pwd)"/creds.txt + EOF + chmod 755 creds.sh && + git -C "$REPO_T1" config --local credential.helper "!f() { cat \"$(pwd)\"/creds.txt; }; f" && + git -C "$REPO_T2" config --local credential.helper "!f() { cat \"$(pwd)\"/creds.txt; }; f" && + # + # Create some test data sets. + # + get_list_of_oids 30 && + get_list_of_commit_and_tree_oids 30 && + get_list_of_blobs_oids && + get_one_commit_oid +' + +# Stop a gvfs-protocol server. +# Usage: stop_gvfs_protocol_server [] +# +# The optional port_increment (default 0) specifies which server to stop. +# Increment 0 uses the base port, 1 uses base+1, etc. +# +stop_gvfs_protocol_server () { + local instance="${1:-0}" + local pid_file="$(server_pid_file "$instance")" + local log_file="$(server_log_file "$instance")" + + if ! test -f "$pid_file" + then + return 0 + fi + # + # The server will shutdown automatically when we delete the pid-file. + # + rm -f "$pid_file" + # + # Give it a few seconds to shutdown (mainly to completely release the + # port before the next test start another instance and it attempts to + # bind to it). + # + for k in $(test_seq 5) + do + if grep -q "Starting graceful shutdown" "$log_file" + then + return 0 + fi + sleep 1 + done + + echo "stop_gvfs_protocol_server($instance): timeout waiting for server shutdown" + return 1 +} + +# Start a gvfs-protocol server. +# Usage: start_gvfs_protocol_server [] +# +# The optional port_increment (default 0) specifies which server to start. +# Increment 0 uses the base port, 1 uses base+1, etc. +# This allows running multiple servers simultaneously on different ports. +# +start_gvfs_protocol_server () { + local instance="${1:-0}" + local port="$(server_port "$instance")" + local pid_file="$(server_pid_file "$instance")" + local log_file="$(server_log_file "$instance")" + # + # Launch our server into the background in repo_src. + # + ( + cd "$REPO_SRC" + test-gvfs-protocol --verbose \ + --listen=127.0.0.1 \ + --port=$port \ + --reuseaddr \ + --pid-file="$pid_file" \ + 2>"$log_file" & + ) + # + # Give it a few seconds to get started. + # + for k in $(test_seq 5) + do + if test -f "$pid_file" + then + return 0 + fi + sleep 1 + done + + echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" + return 1 +} + +start_gvfs_protocol_server_with_mayhem () { + if test $# -lt 1 + then + echo "start_gvfs_protocol_server_with_mayhem: need mayhem args" + return 1 + fi + + mayhem="" + for k in $* + do + mayhem="$mayhem --mayhem=$k" + done + # + # Launch our server into the background in repo_src. + # + ( + cd "$REPO_SRC" + test-gvfs-protocol --verbose \ + --listen=127.0.0.1 \ + --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ + --reuseaddr \ + --pid-file="$PID_FILE" \ + $mayhem \ + 2>"$SERVER_LOG" & + ) + # + # Give it a few seconds to get started. + # + for k in $(test_seq 5) + do + if test -f "$PID_FILE" + then + return 0 + fi + sleep 1 + done + + echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" + return 1 +} + +# Verify that a server received at least one connection. +# Usage: verify_server_was_contacted [] +# +verify_server_was_contacted () { + local instance="${1:-0}" + local log_file="$(server_log_file "$instance")" + grep -q "Connection from" "$log_file" +} + +# Verify that a server was NOT contacted. +# Usage: verify_server_was_not_contacted [] +# +verify_server_was_not_contacted () { + local instance="${1:-0}" + local log_file="$(server_log_file "$instance")" + ! grep -q "Connection from" "$log_file" +} + +# Verify the number of connections from the client. +# +# If keep-alive is working, a series of successful sequential requests to the +# same server should use the same TCP connection, so a simple multi-get would +# only have one connection. +# +# On the other hand, an auto-retry after a network error (mayhem) will have +# more than one for a single object request. +# +# TODO This may generate false alarm when we get to complicated tests, so +# TODO we might only want to use it for basic tests. +# +verify_connection_count () { + if test $# -eq 1 + then + expected_nr=$1 + else + expected_nr=1 + fi + + actual_nr=$(grep -c "Connection from" "$SERVER_LOG") + + if test $actual_nr -ne $expected_nr + then + echo "verify_keep_live: expected $expected_nr; actual $actual_nr" + return 1 + fi + return 0 +} + +# Verify that the set of requested objects are present in +# the shared-cache and that there is no corruption. We use +# cat-file to hide whether the object is packed or loose in +# the test repo. +# +# Usage: +# +verify_objects_in_shared_cache () { + # + # See if any of the objects are missing from repo_t1. + # + git -C "$REPO_T1" cat-file --batch-check <"$1" >OUT.bc_actual || return 1 + test_grep " missing" OUT.bc_actual && return 1 + # + # See if any of the objects have different sizes or types than repo_src. + # + git -C "$REPO_SRC" cat-file --batch-check <"$1" >OUT.bc_expect || return 1 + test_cmp OUT.bc_expect OUT.bc_actual || return 1 + # + # See if any of the objects are corrupt in repo_t1. This fully + # reconstructs the objects and verifies the hash and therefore + # detects corruption not found by the earlier "batch-check" step. + # + git -C "$REPO_T1" cat-file --batch <"$1" >OUT.b_actual || return 1 + # + # TODO move the shared-cache directory (and/or the + # TODO .git/objects/info/alternates and temporarily unset + # TODO gvfs.sharedCache) and repeat the first "batch-check" + # TODO and make sure that they are ALL missing. + # + return 0 +} + +# gvfs-helper prints a "packfile " message for each received +# packfile to stdout. Verify that we received the expected number +# of packfiles. +# +verify_received_packfile_count () { + if test $# -eq 1 + then + expected_nr=$1 + else + expected_nr=1 + fi + + actual_nr=$(grep -c "packfile " /dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + echo dead data walking >empty-in-repo && + test_must_fail git add empty-in-repo +' + +test_expect_success "filter: smudge filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + test_must_fail git checkout && + + # ensure the local core.gvfs setting overwrites the global setting + git config --global core.gvfs false && + test_must_fail git checkout +' + +test_expect_success "ident blocked on add when under GVFS" ' + test_config core.gvfs 64 && + test_config core.autocrlf false && + + echo "*.i ident" >.gitattributes && + echo "\$Id\$" > ident.i && + + test_must_fail git add ident.i +' + +test_expect_success "ident blocked when under GVFS" ' + git add ident.i && + + git commit -m "added ident.i" && + test_config core.gvfs 64 && + rm ident.i && + + test_must_fail git checkout -- ident.i +' + test_expect_success 'disable filter with empty override' ' test_config_global filter.disable.smudge false && test_config_global filter.disable.clean false && diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index 49dbf09da77386..8f42f28f56ab43 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -343,6 +343,18 @@ checkout_files () { " } +test_expect_success 'crlf conversions blocked when under GVFS' ' + git checkout -b gvfs && + test_commit initial && + rm initial.t && + test_config core.gvfs 64 && + test_config core.autocrlf true && + test_must_fail git read-tree --reset -u HEAD && + + git config core.autocrlf false && + git read-tree --reset -u HEAD +' + # Test control characters # NUL SOH CR EOF==^Z test_expect_success 'ls-files --eol -o Text/Binary' ' diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 5abfa202c19dca..2c805f85140c24 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -623,7 +623,7 @@ test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD '%(prefix)/ works' test_expect_success MINGW,RUNTIME_PREFIX 'MSYSTEM/PATH is adjusted if necessary' ' if test -z "$MINGW_PREFIX" then - MINGW_PREFIX="/$(echo "${MSYSTEM:-MINGW64}" | tr A-Z a-z)" + MINGW_PREFIX="/$(echo "${MSYSTEM:-UCRT64}" | tr A-Z a-z)" fi && mkdir -p "$HOME"/bin pretend"$MINGW_PREFIX"/bin \ pretend"$MINGW_PREFIX"/libexec/git-core pretend/usr/bin && diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh new file mode 100755 index 00000000000000..34aed1711459bd --- /dev/null +++ b/t/t0400-pre-command-hook.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +test_description='pre-command hook' + +. ./test-lib.sh + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/pre-command <<-EOF && + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/pre-command.out + EOF + echo "second" >> file && + git add file && + test "add file" = "$(cat .git/pre-command.out)" && + echo Hello | git hash-object --stdin && + test "hash-object --stdin" = "$(cat .git/pre-command.out)" +' + +test_expect_success 'with failing hook' ' + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/pre-command.out)" +' + +test_expect_success 'in a subdirectory' ' + echo touch i-was-here | write_script .git/hooks/pre-command && + mkdir sub && + ( + cd sub && + git version + ) && + test_path_is_file sub/i-was-here +' + +test_expect_success 'in a subdirectory, using an alias' ' + git reset --hard && + echo "echo \"\$@; \$(pwd)\" >>log" | + write_script .git/hooks/pre-command && + mkdir -p sub && + ( + cd sub && + git -c alias.v="version" v + ) && + test_path_is_missing log && + test_line_count = 2 sub/log +' + +test_expect_success 'with core.hooksPath' ' + mkdir -p .git/alternateHooks && + write_script .git/alternateHooks/pre-command <<-EOF && + echo "alternate" >\$(git rev-parse --git-dir)/pre-command.out + EOF + write_script .git/hooks/pre-command <<-EOF && + echo "original" >\$(git rev-parse --git-dir)/pre-command.out + EOF + git -c core.hooksPath=.git/alternateHooks status && + test "alternate" = "$(cat .git/pre-command.out)" +' +test_done diff --git a/t/t0401-post-command-hook.sh b/t/t0401-post-command-hook.sh new file mode 100755 index 00000000000000..41cbbbc2c6ba10 --- /dev/null +++ b/t/t0401-post-command-hook.sh @@ -0,0 +1,127 @@ +#!/bin/sh + +test_description='post-command hook' + +. ./test-lib.sh + +test_expect_success 'hook does not block git help' ' + git config help.autocorrect immediate && + git commit --allow-empty -m "a single log entry" && + mkdir -p .git/hooks && + write_script .git/hooks/post-command <<-EOF && + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/post-command.out + EOF + # intentional typo "logg" gets autocorrected to "log" + git logg --format=%s --first-parent > actual && + test "log --format=%s --first-parent --exit_code=0" = "$(cat .git/post-command.out)" && + echo "a single log entry" >expect && + test_cmp expect actual +' + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/post-command <<-EOF && + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/post-command.out + EOF + echo "second" >> file && + git add file && + test "add file --exit_code=0" = "$(cat .git/post-command.out)" +' + +test_expect_success 'with failing pre-command hook' ' + test_when_finished rm -f .git/hooks/pre-command && + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/post-command.out)" +' + +test_expect_success 'with post-index-change config' ' + mkdir -p internal-hooks && + write_script internal-hooks/post-command <<-EOF && + echo ran >post-command.out + EOF + write_script internal-hooks/post-index-change <<-EOF && + echo ran >post-index-change.out + EOF + + # prevent writing of sentinel files to this directory. + test_when_finished chmod 775 internal-hooks && + chmod a-w internal-hooks && + + git config core.hooksPath internal-hooks && + + # First, show expected behavior. + echo ran >expect && + rm -f post-command.out post-index-change.out && + + # rev-parse leaves index intact, but runs post-command. + git rev-parse HEAD && + test_path_is_missing post-index-change.out && + test_cmp expect post-command.out && + rm -f post-command.out && + + echo stuff >>file && + # add updates the index and runs post-command. + git add file && + test_cmp expect post-index-change.out && + test_cmp expect post-command.out && + + # Now, show configured behavior + git config postCommand.strategy worktree-change && + + # rev-parse leaves index intact and thus skips post-command. + rm -f post-command.out post-index-change.out && + git rev-parse HEAD && + test_path_is_missing post-index-change.out && + test_path_is_missing post-command.out && + + echo stuff >>file && + # add keeps the worktree the same, so does not run post-command. + rm -f post-command.out post-index-change.out && + git add file && + test_cmp expect post-index-change.out && + test_path_is_missing post-command.out && + + # add keeps the worktree the same, so does not run post-command. + # and this should work through an alias. + git config alias.addalias add && + rm -f post-command.out post-index-change.out && + echo more stuff >>file && + git addalias file && + test_cmp expect post-index-change.out && + test_path_is_missing post-command.out && + + echo stuff >>file && + # reset --hard updates the worktree. + # even through an alias + git config alias.resetalias "reset --hard" && + rm -f post-command.out post-index-change.out && + git resetalias && + test_cmp expect post-index-change.out && + test_cmp expect post-command.out && + + rm -f post-command.out && + test_must_fail git && # get help text + test_path_is_missing post-command.out && + + rm -f post-command.out && + git version && + test_path_is_missing post-command.out && + + rm -f post-command.out && + test_must_fail git typo && + test_path_is_missing post-command.out +' + +test_done diff --git a/t/t0402-block-command-on-gvfs.sh b/t/t0402-block-command-on-gvfs.sh new file mode 100755 index 00000000000000..2d688986675f0f --- /dev/null +++ b/t/t0402-block-command-on-gvfs.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +test_description='block commands in GVFS repo' + +. ./test-lib.sh + +not_with_gvfs () { + command=$1 && + shift && + test_expect_success "test $command $*" " + test_config alias.g4rbled $command && + test_config core.gvfs true && + test_must_fail git $command $* && + test_must_fail git g4rbled $* && + test_unconfig core.gvfs && + test_must_fail git -c core.gvfs=true $command $* && + test_must_fail git -c core.gvfs=true g4rbled $* + " +} + +not_with_gvfs fsck +not_with_gvfs gc +not_with_gvfs gc --auto +not_with_gvfs prune +not_with_gvfs submodule status +not_with_gvfs update-index --index-version 2 +not_with_gvfs update-index --skip-worktree +not_with_gvfs update-index --no-skip-worktree +not_with_gvfs update-index --split-index + +# worktree is conditionally allowed: blocked when VFS enabled without +# GVFS_SUPPORTS_WORKTREES. +test_expect_success 'worktree blocked with VFS but without SUPPORTS_WORKTREES' ' + test_config core.gvfs $((0xffff & ~(1<<8))) && # all bits except GVFS_SUPPORTS_WORKTREES + test_must_fail git worktree list 2>err && + test_grep "not supported when using the virtual file system" err +' + +test_expect_success 'worktree operations work when SUPPORTS_WORKTREES is set' ' + test_commit initial && + + # Use core.gvfs=true which sets all bits including SUPPORTS_WORKTREES. + test_config core.gvfs true && + + # add: succeeds, forces --no-checkout (no initial.t on disk) + git worktree add ../vfs-wt && + test_path_exists ../vfs-wt/.git && + ! test_path_exists ../vfs-wt/initial.t && + + # list: shows the worktree + git worktree list >out && + test_grep "vfs-wt" out && + + # remove: cleans up + git worktree remove --force ../vfs-wt && + ! test_path_exists ../vfs-wt +' + +test_expect_success 'test gc --auto succeeds when disabled via config' ' + test_config core.gvfs true && + test_config gc.auto 0 && + git gc --auto +' + +test_expect_success 'test repack fails with VFS bit enabled' ' + test_config core.gvfs true && + test_must_fail git repack +' + +test_expect_success 'test repack succeeds with VFS bit disabled' ' + test_config core.gvfs 150 && + git repack +' + +test_done diff --git a/t/t0410/read-object b/t/t0410/read-object new file mode 100755 index 00000000000000..02c799837f4057 --- /dev/null +++ b/t/t0410/read-object @@ -0,0 +1,118 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "../.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40,64})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + + open my $log, '>>.git/read-object-hook.log'; + print $log "Read object $sha1, exit code $?\n"; + close $log; + } else { + die "bad command '$command'"; + } +} diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh new file mode 100755 index 00000000000000..5ae8ca2676f3b4 --- /dev/null +++ b/t/t0499-read-object.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +test_description='tests for long running read-object process' + +. ./test-lib.sh + +test_expect_success 'setup host repo with a root commit' ' + test_commit zero && + hash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\ -f3) +' + +test_expect_success 'blobs can be retrieved from the host repo' ' + git init guest-repo && + (cd guest-repo && + mkdir -p .git/hooks && + sed "1s|/usr/bin/perl|$PERL_PATH|" \ + <$TEST_DIRECTORY/t0410/read-object \ + >.git/hooks/read-object && + chmod +x .git/hooks/read-object && + git config core.virtualizeobjects true && + git cat-file blob "$hash1") +' + +test_expect_success 'invalid blobs generate errors' ' + (cd guest-repo && + test_must_fail git cat-file blob "invalid") +' + +test_expect_success 'read-object-hook is bypassed when writing objects' ' + (cd guest-repo && + echo hello >hello.txt && + git add hello.txt && + hash="$(git rev-parse --verify :hello.txt)" && + test_grep ! "$hash" .git/read-object-hook.log) +' + +test_expect_success 'setup no-fetch commit lookups' ' + git init no-fetch && + test_commit -C no-fetch --no-tag parent && + parent=$(git -C no-fetch rev-parse HEAD) && + test_commit -C no-fetch --no-tag tip && + tip=$(git -C no-fetch rev-parse HEAD) && + parent_path=no-fetch/.git/objects/$(test_oid_to_path "$parent") && + mv "$parent_path" parent-object && + mkdir -p no-fetch/.git/hooks && + write_script no-fetch/.git/hooks/read-object <<-\EOF + echo invoked >hook-called + exit 1 + EOF +' + +test_expect_success 'no-fetch commit lookups skip object acquisition' ' + test_write_lines "$tip" "?$parent" >expect && + for helper in false true + do + env GIT_TEST_COMMIT_GRAPH=0 \ + GIT_TRACE2_EVENT="$PWD/no-fetch-$helper.trace" \ + git -C no-fetch -c core.gvfs=0 -c core.commitGraph=false \ + -c core.useGVFSHelper=$helper \ + -c core.virtualizeObjects=true \ + rev-list --missing=print HEAD >actual && + test_cmp expect actual && + test_path_is_missing no-fetch/hook-called && + test_grep ! child_start "no-fetch-$helper.trace" || + return 1 + done +' + +test_expect_success 'no-fetch commit lookups still report corruption' ' + : >"$parent_path" && + for helper in false true + do + test_must_fail \ + env GIT_TEST_COMMIT_GRAPH=0 \ + GIT_TRACE2_EVENT="$PWD/corrupt-$helper.trace" \ + git -C no-fetch \ + -c core.gvfs=0 -c core.commitGraph=false \ + -c core.useGVFSHelper=$helper \ + -c core.virtualizeObjects=true \ + rev-list --missing=print HEAD >actual 2>err && + test_grep "fatal: loose object .* is corrupt" err && + test_path_is_missing no-fetch/hook-called && + test_grep ! child_start "corrupt-$helper.trace" || + return 1 + done +' + +test_done diff --git a/t/t1018-read-tree-skip-sha-on-read.sh b/t/t1018-read-tree-skip-sha-on-read.sh new file mode 100755 index 00000000000000..5b76a80a0020dc --- /dev/null +++ b/t/t1018-read-tree-skip-sha-on-read.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='check that read-tree works with core.gvfs config value' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-read-tree.sh + +test_expect_success setup ' + echo one >a && + git add a && + git commit -m initial +' +test_expect_success 'read-tree without core.gvsf' ' + read_tree_u_must_succeed -m -u HEAD +' + +test_expect_success 'read-tree with core.gvfs set to 1' ' + git config core.gvfs 1 && + read_tree_u_must_succeed -m -u HEAD +' + +test_done diff --git a/t/t1060-object-corruption.sh b/t/t1060-object-corruption.sh index d2ef468b4528ea..3bb56bb7ba8171 100755 --- a/t/t1060-object-corruption.sh +++ b/t/t1060-object-corruption.sh @@ -3,6 +3,7 @@ test_description='see how we handle various forms of corruption' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-diff-data.sh # convert "1234abcd" to ".git/objects/12/34abcd" obj_to_file() { @@ -62,6 +63,35 @@ test_expect_success 'streaming a corrupt blob fails' ' ) ' +test_expect_success PERL 'truncated objects can be re-retrieved via GVFS' ' + git init truncated && + COPYING_test_data >truncated/COPYING && + git -C truncated add COPYING && + test_tick && + git -C truncated commit -m initial COPYING && + + # set up the `read-object` hook so that it overwrites the corrupt object + mkdir -p truncated/.git/hooks && + sed -e "1s|/usr/bin/perl|$PERL_PATH|" \ + -e "s/system/unlink \".git\/objects\/\" . substr(\$sha1, 0, 2) . \"\/\" . substr(\$sha1, 2); &/" \ + <$TEST_DIRECTORY/t0410/read-object \ + >truncated/.git/hooks/read-object && + chmod +x truncated/.git/hooks/read-object && + + # ensure that the parent repository has a copy of the object, from + # where the `read-object` can read it + sha="$(git hash-object -w truncated/COPYING)" && + file=$(obj_to_file $sha) && + size=$(test_file_size $file) && + chmod u+w truncated/$file && + test-tool truncate truncated/$file $(($size-8)) && + + rm truncated/COPYING && + test_must_fail git -C truncated reset --hard && + git -C truncated -c core.gvfs=4 -c core.virtualizeObjects \ + reset --hard +' + test_expect_success 'getting type of a corrupt blob fails' ' ( cd bit-error && diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 529844e2862c74..02b393e36a7d28 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -106,6 +106,26 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs test_cmp expect actual ' +test_expect_success 'checkout does not delete items outside the sparse checkout file' ' + # The "core.virtualfilesystem" config will prevent the + # SKIP_WORKTREE flag from being dropped on files present on-disk. + test_config core.virtualfilesystem true && + + test_config core.gvfs 8 && + git checkout -b outside && + echo "new file1" >d && + git add --sparse d && + git commit -m "branch initial" && + echo "new file1" >e && + git add --sparse e && + git commit -m "skipped worktree" && + git update-index --skip-worktree e && + echo "/d" >.git/info/sparse-checkout && + git checkout HEAD^ && + test_path_is_file d && + test_path_is_file e +' + test_expect_success MINGW 'no unnecessary opendir() with fscache' ' git clone . fscache-test && ( diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 74b1761e0c8507..148a86f9df207e 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -701,6 +701,7 @@ test_expect_success 'pattern-checks: contained glob characters' ' test_expect_success BSLASHPSPEC 'pattern-checks: escaped characters' ' git clone repo escaped && + git -C escaped config advice.sparseIndexExpanded false && TREEOID=$(git -C escaped rev-parse HEAD:folder1) && NEWTREE=$(git -C escaped mktree <<-EOF $(git -C escaped ls-tree HEAD) @@ -782,6 +783,10 @@ test_expect_success 'cone mode clears ignored subdirectories' ' git -C repo status --porcelain=v2 >out && test_must_be_empty out && + git -C repo -c index.deleteSparseDirectories=false sparse-checkout reapply && + test_path_is_dir repo/folder1 && + test_path_is_dir repo/deep/deeper2 && + git -C repo sparse-checkout reapply && test_path_is_missing repo/folder1 && test_path_is_missing repo/deep/deeper2 && diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 05b54062b3bc85..a64cdcec91a2fe 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -155,6 +155,7 @@ init_repos () { git -C sparse-index reset --hard && # initialize sparse-checkout definitions + git -C sparse-checkout config index.sparse false && git -C sparse-checkout sparse-checkout init --cone && git -C sparse-checkout sparse-checkout set deep && git -C sparse-index sparse-checkout init --cone --sparse-index && @@ -317,6 +318,22 @@ test_expect_success 'root directory cannot be sparse' ' test_cmp expect actual ' +test_expect_success 'sparse-checkout with untracked files and dirs' ' + init_repos && + + # Empty directories outside sparse cone are deleted + run_on_sparse mkdir -p deep/empty && + test_sparse_match git sparse-checkout set folder1 && + test_must_be_empty sparse-checkout-err && + run_on_sparse test_path_is_missing deep && + + # Untracked files outside sparse cone are not deleted + run_on_sparse touch folder1/another && + test_sparse_match git sparse-checkout set folder2 && + test_grep "directory ${SQ}folder1/${SQ} contains untracked files" sparse-checkout-err && + run_on_sparse test_path_exists folder1/another +' + test_expect_success 'status with options' ' init_repos && test_sparse_match ls && @@ -610,6 +627,45 @@ test_expect_success 'diff --cached' ' test_all_match git diff --cached ' +test_expect_success 'diff partially-staged' ' + init_repos && + + git -C full-checkout config advice.sparseIndexExpanded false && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + # Add file within cone + test_all_match git sparse-checkout set deep && + run_on_all ../edit-contents deep/testfile && + test_all_match git add deep/testfile && + run_on_all ../edit-contents deep/testfile && + + test_all_match git diff && + test_all_match git diff --staged && + + # Add file outside cone + test_all_match git reset --hard && + run_on_all mkdir newdirectory && + run_on_all ../edit-contents newdirectory/testfile && + test_all_match git sparse-checkout set newdirectory && + test_all_match git add newdirectory/testfile && + run_on_all ../edit-contents newdirectory/testfile && + test_all_match git sparse-checkout set && + + test_all_match git diff && + test_all_match git diff --staged && + + # Merge conflict outside cone + test_all_match git reset --hard && + test_all_match git checkout merge-left && + test_all_match test_must_fail git merge merge-right && + + test_all_match git diff && + test_all_match git diff --staged +' + # NEEDSWORK: sparse-checkout behaves differently from full-checkout when # running this test with 'df-conflict-2' after 'df-conflict-1'. test_expect_success 'diff with renames and conflicts' ' @@ -1090,7 +1146,9 @@ test_expect_success 'read-tree --merge with directory-file conflicts' ' test_expect_success 'merge, cherry-pick, and rebase' ' init_repos && - for OPERATION in "merge -m merge" cherry-pick "rebase --apply" "rebase --merge" + # microsoft/git specific: we need to use "quiet" mode + # to avoid different stderr for some rebases. + for OPERATION in "merge -m merge" cherry-pick "rebase -q --apply" "rebase -q --merge" do test_all_match git checkout -B temp update-deep && test_all_match git $OPERATION update-folder1 && @@ -1588,6 +1646,11 @@ test_expect_success 'sparse-index is not expanded' ' ensure_not_expanded reset --merge update-deep && ensure_not_expanded reset --hard && + echo a test change >>sparse-index/README.md && + ensure_not_expanded diff && + git -C sparse-index add README.md && + ensure_not_expanded diff --staged && + ensure_not_expanded reset base -- deep/a && ensure_not_expanded reset base -- nonexistent-file && ensure_not_expanded reset deepest -- deep && @@ -1975,6 +2038,46 @@ test_expect_success 'sparse index is not expanded: sparse-checkout' ' ensure_not_expanded sparse-checkout set ' +# NEEDSWORK: although the full repository's index is _not_ expanded as part of +# stash, a temporary index, which is _not_ sparse, is created when stashing and +# applying a stash of untracked files. As a result, the test reports that it +# finds an instance of `ensure_full_index`, but it does not carry with it the +# performance implications of expanding the full repository index. +test_expect_success 'sparse index is not expanded: stash -u' ' + init_repos && + + mkdir -p sparse-index/folder1 && + echo >>sparse-index/README.md && + echo >>sparse-index/a && + echo >>sparse-index/folder1/new && + + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index stash -u && + test_region index ensure_full_index trace2.txt && + + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index stash pop && + test_region index ensure_full_index trace2.txt +' + +# NEEDSWORK: similar to `git add`, untracked files outside of the sparse +# checkout definition are successfully stashed and unstashed. +test_expect_success 'stash -u outside sparse checkout definition' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + run_on_sparse mkdir -p folder1 && + run_on_all ../edit-contents folder1/new && + test_all_match git stash -u && + test_all_match git status --porcelain=v2 && + + test_all_match git stash pop -q && + test_all_match git status --porcelain=v2 +' + # NEEDSWORK: a sparse-checkout behaves differently from a full checkout # in this scenario, but it shouldn't. test_expect_success 'reset mixed and checkout orphan' ' @@ -2744,4 +2847,20 @@ test_expect_success 'sparse-index is not expanded: restore --source --staged' ' ensure_not_expanded restore --source update-folder1 --staged . ' +test_expect_success 'ensure_full_index_with_reason' ' + init_repos && + + GIT_TRACE2_EVENT="$(pwd)/ls-files-trace" \ + git -C sparse-index ls-files --no-sparse HEAD && + test_trace2_data "sparse-index" "expansion-reason" "ls-files" sparse-index/folder2/a && + GIT_TRACE2_EVENT="$(pwd)/status-trace" \ + git -C sparse-index status && + test_trace2_data "sparse-index" "skip-worktree sparsedir" "folder2/" .gitignore <<-\EOF && + .gitignore + expect* + actual* + EOF + mkdir -p dir1 && + touch dir1/file1.txt && + touch dir1/file2.txt && + mkdir -p dir2 && + touch dir2/file1.txt && + touch dir2/file2.txt && + git add . && + git commit -m "initial" && + git config --local core.virtualfilesystem .git/hooks/virtualfilesystem +' + +test_expect_success 'test hook parameters and version' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + if test "$#" -ne 1 + then + echo "$0: Exactly 1 argument expected" >&2 + exit 2 + fi + + if test "$1" != 1 + then + echo "$0: Unsupported hook version." >&2 + exit 1 + fi + EOF + git status && + write_script .git/hooks/virtualfilesystem <<-\EOF && + exit 3 + EOF + test_must_fail git status +' + +test_expect_success 'verify status is clean' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + EOF + rm -f .git/index && + git checkout -f && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + git status > actual && + cat > expected <<-\EOF && + On branch main + nothing to commit, working tree clean + EOF + test_cmp expected actual +' + +test_expect_success 'verify skip-worktree bit is set for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + S dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify skip-worktree bit is cleared for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file2.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folder wild cards' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folders not included are ignored' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify including one file doesnt include the rest' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir1/dir2/a\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/dir2/a + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify files not listed are ignored by git clean -f -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test -d dir3 && + test -f dir3/untracked.txt +' + +test_expect_success 'verify files not listed are ignored by git clean -f -d -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + printf "dir3/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -d -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test ! -d dir3 && + test ! -f dir3/untracked.txt +' + +test_expect_success 'verify folder entries include all files' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/b + ?? dir1/dir2/a + ?? dir1/dir2/b + ?? dir1/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify case insensitivity of virtual file system entries' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/a\0" + printf "Dir1/Dir2/a\0" + printf "DIR2/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git -c core.ignorecase=false status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + EOF + test_cmp expected actual && + git -c core.ignorecase=true status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/dir2/a + ?? dir2/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file3.txt\0" + EOF + touch dir1/file3.txt && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + S dir1/file2.txt + H dir1/file3.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file3.txt\0" + EOF + mv dir1/file1.txt dir1/file3.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + ?? dir1/file3.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file deleted' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + rm dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file overwritten' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + echo "overwritten" > dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + M dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on folder created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/dir1/\0" + EOF + mkdir -p dir1/dir1 && + git status -su > actual && + cat > expected <<-\EOF && + EOF + test_cmp expected actual && + git clean -fd && + test ! -d "/dir1/dir1" +' + +test_expect_success 'on folder renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir3/\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir3/file1.txt\0" + printf "dir3/file2.txt\0" + EOF + mv dir1 dir3 && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + D dir1/file2.txt + ?? dir3/file1.txt + ?? dir3/file2.txt + ?? dir3/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'folder with same prefix as file' ' + clean_repo && + touch dir1.sln && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + printf "dir1.sln\0" + EOF + git add dir1.sln && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1.sln + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'checkout skips lstat for deleted skip-worktree entries in VFS mode' ' + # When switching branches, entries present in the old tree but absent + # in the new tree go through deleted_entry() -> verify_absent_if_directory(). + # Without the fix, the tree entry lacks CE_NEW_SKIP_WORKTREE (only + # src_index entries get that flag), so verify_absent_if_directory() + # falls through to verify_absent_1() which lstats the path. If a + # directory exists where the deleted file entry was (simulating a + # worst-case scenario), the lstat finds it and + # verify_clean_subdirectory() rejects the checkout due to untracked + # content inside. + # + # With the fix, verify_absent_if_directory() is skipped entirely + # when VFS mode is active — no lstat, no rejection, checkout completes. + # + # Set up two branches: main has dir1/ + dir2/, side has only dir1/ + clean_repo && + + test_when_finished "rm -rf dir2/file1.txt && git -c core.virtualfilesystem= checkout main" && + + git -c core.virtualfilesystem= checkout -b side && + git -c core.virtualfilesystem= rm -rf dir2 && + git -c core.virtualfilesystem= commit -m "remove dir2" && + git -c core.virtualfilesystem= checkout main && + + # Configure VFS hook that returns nothing (0% hydration) + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "" + EOF + + # Create a directory where the deleted file entry is, with + # untracked content inside. This would not happen with a real + # VFS because the VFS would report the file-to-directory change + # in the virtualfilesystem hook results, clearing skip-worktree. + # But it lets us verify that the lstat is not called: without + # the fix, verify_absent_1() lstats this path, finds a directory, + # and verify_clean_subdirectory() rejects the checkout because of + # the untracked file inside. + rm -f dir2/file1.txt && + mkdir -p dir2/file1.txt && + echo "untracked" >dir2/file1.txt/trap.txt && + + # Verify all entries are skip-worktree before checkout + git ls-files -v >actual && + test_grep ! "^H " actual && + + # Checkout to side branch. Without the fix this fails because + # verify_absent_1 finds untracked content in the directory at + # dir2/file1.txt. With the fix the lstat is skipped entirely. + git checkout side +' + +test_expect_success 'checkout -- preserves skip-worktree in VFS mode' ' + # When "git checkout -- " updates the index with a + # different version of a file, update_some() creates a replacement + # cache entry. Without the fix, skip-worktree is cleared on the + # new entry, causing checkout_entry() to try unlink() + write on + # disk. For virtual files with no physical NTFS entry, the unlink + # fails with ENOENT and the command exits 255. + # + # With the fix, skip-worktree is preserved from the old index + # entry when core_virtualfilesystem is set. The index is updated + # to the tree entry OID, but checkout_entry() is skipped entirely. + clean_repo && + + test_when_finished "git -c core.virtualfilesystem= checkout main" && + + # Create a second commit with modified content + git -c core.virtualfilesystem= checkout -b checkout-path-test && + echo "modified content" >dir1/file1.txt && + git -c core.virtualfilesystem= add dir1/file1.txt && + git -c core.virtualfilesystem= commit -m "modify dir1/file1.txt" && + + # Record the OIDs for verification + git rev-parse HEAD:dir1/file1.txt >expect_new_oid && + git rev-parse HEAD~1:dir1/file1.txt >expect_old_oid && + + # Configure VFS hook that returns nothing (0% hydration). + # All entries keep skip-worktree set, simulating virtual files + # with no physical on-disk representation. + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "" + EOF + + # Remove the physical file to simulate a virtual placeholder. + # With the fix, checkout should update the index without + # touching the working tree (no file creation). + # Without the fix, checkout would clear skip-worktree and + # write the file to disk. + rm -f dir1/file1.txt && + + # Checkout the old version of the file from the parent commit. + git checkout HEAD~1 -- dir1/file1.txt && + + # Index should have the old (HEAD~1) OID + git ls-files -s dir1/file1.txt >actual_index && + test_grep "$(cat expect_old_oid)" actual_index && + + # The file should NOT have been written to disk — the fix + # preserves skip-worktree so checkout_entry() is skipped. + # Without the fix, checkout clears skip-worktree and writes + # the file to disk. + test_path_is_missing dir1/file1.txt +' + +test_expect_success MINGW,FSMONITOR_DAEMON 'virtualfilesystem hook disables built-in FSMonitor' ' + clean_repo && + test_config core.usebuiltinfsmonitor true && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + git config core.virtualfilesystem .git/hooks/virtualfilesystem && + git status && + test_must_fail git fsmonitor--daemon status +' + +# reset --mixed tests for virtual filesystem mode +# +# Background: reset --mixed moves HEAD and updates the index to match the +# target commit, but leaves the working tree untouched. In a normal repo, +# files whose index entry changed show as "unstaged changes" because the +# working tree still has the pre-reset content. +# +# In VFS mode, most files have skip-worktree set and don't exist on disk. +# The VFS-specific code in update_index_from_diff() handles this: +# +# - Files NOT on disk (virtual/placeholder): git writes the pre-reset +# content to disk via checkout_entry() and clears skip-worktree, so +# they correctly appear as "modified" in status. +# +# - Files already on disk (hydrated by a previous read): their on-disk +# content is still the pre-reset version. Skip-worktree must also be +# cleared for these so refresh_index() compares them against the new +# index entry and reports them as modified. +# +# A bug existed where hydrated files (file_exists returns true) kept +# skip-worktree set after the reset, making them invisible to status. + +test_expect_success 'reset --mixed reports hydrated files as modified in VFS mode' ' + clean_repo && + + # Create a second commit that modifies dir1/file1.txt + git -c core.virtualfilesystem= checkout -b reset-hydrated-test && + echo "modified content" >dir1/file1.txt && + git -c core.virtualfilesystem= add dir1/file1.txt && + git -c core.virtualfilesystem= commit -m "modify dir1/file1.txt" && + + # VFS hook: nothing in ModifiedPaths — all files get skip-worktree + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "" + EOF + + # dir1/file1.txt is on disk with the new commit content (simulates + # a hydrated file that was read but never written to ModifiedPaths). + # file_exists() will return true for it. + test_path_is_file dir1/file1.txt && + + # reset --mixed to parent: index moves to old content, working tree + # keeps new content. dir1/file1.txt should show as modified in + # the reset output because skip-worktree is cleared during the + # reset so refresh_index detects the mismatch. + git reset HEAD~1 >actual_stdout && + test_grep "dir1/file1.txt" actual_stdout +' + +test_expect_success 'reset --mixed reports non-hydrated files as modified in VFS mode' ' + clean_repo && + + # Create a second commit that modifies dir1/file1.txt + git -c core.virtualfilesystem= checkout -b reset-virtual-test && + echo "modified content" >dir1/file1.txt && + git -c core.virtualfilesystem= add dir1/file1.txt && + git -c core.virtualfilesystem= commit -m "modify dir1/file1.txt" && + + # VFS hook: nothing in ModifiedPaths + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "" + EOF + + # Remove the file from disk to simulate a non-hydrated virtual file. + # file_exists() will return false for it. + rm -f dir1/file1.txt && + + # reset --mixed to parent: git should write pre-reset content to + # disk and clear skip-worktree, reporting the file as modified. + git reset HEAD~1 >actual_stdout && + test_grep "dir1/file1.txt" actual_stdout && + + # The pre-reset content should have been written to disk + test_path_is_file dir1/file1.txt +' + +test_expect_success 'reset --mixed with hydrated file leaves other skip-worktree intact' ' + clean_repo && + + # Create a commit that modifies dir1/file1.txt but NOT dir2/file1.txt + git -c core.virtualfilesystem= checkout -b reset-partial-test && + echo "modified content" >dir1/file1.txt && + git -c core.virtualfilesystem= add dir1/file1.txt && + git -c core.virtualfilesystem= commit -m "modify dir1/file1.txt only" && + + # VFS hook: nothing in ModifiedPaths + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "" + EOF + + # Reset: only dir1/file1.txt changed between HEAD and HEAD~1. + # dir2/file1.txt should not appear in the output at all. + git reset HEAD~1 >actual_stdout && + test_grep "dir1/file1.txt" actual_stdout && + test_grep ! "dir2/file1.txt" actual_stdout +' + +test_done diff --git a/t/t1517-outside-repo.sh b/t/t1517-outside-repo.sh index efbac29c0e7075..d01829512baf62 100755 --- a/t/t1517-outside-repo.sh +++ b/t/t1517-outside-repo.sh @@ -130,11 +130,12 @@ do credential-osxkeychain | cvsexportcommit | cvsimport | cvsserver | \ daemon | \ difftool--helper | format-rev | fsck-objects | get-tar-commit-id | \ - gui | gui--askpass | \ + gui | gui--askpass | gvfs-helper | \ http-backend | http-fetch | http-push | init-db | \ mktag | p4 | p4.py | pickaxe | remote-ftp | remote-ftps | \ remote-http | remote-https | replay | send-email | \ sh-i18n--envsubst | shell | show | stage | survey | \ + update-microsoft-git | \ upload-archive--writer | upload-pack | whatchanged) h_expect_outcome=expect_failure all_expect_outcome=expect_failure diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index c2c198c43e9922..de519f5857ed3d 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -21,6 +21,38 @@ object_type_disk_usage() { fi } +check_structure_summary() { + sed -n "1,$(wc -l summary && + test_cmp expect summary +} + +expected_size_histograms() { + git rev-list --all --objects --no-object-names >oids && + git cat-file \ + --batch-check="%(objecttype) %(objectsize) %(objectsize:disk)" \ + sizes && + awk ' + $1 == "tag" { next } + { + bin = 0 + for (size = $2; size >= 16; size = int(size / 16)) + bin++ + key = "objects." $1 "s.histogram.size." bin + count[key]++ + inflated[key] += $2 + disk[key] += $3 + } + END { + for (key in count) { + printf "%s.count=%.0f\n", key, count[key] + printf "%s.inflated_size=%.0f\n", + key, inflated[key] + printf "%s.disk_size=%.0f\n", key, disk[key] + } + } + ' sizes +} + test_expect_success 'empty repository' ' test_when_finished "rm -rf repo" && git init repo && @@ -36,6 +68,16 @@ test_expect_success 'empty repository' ' | * Annotated | 0 | | * Remotes | 0 | | * Others | 0 | + | * Symbolic refs | 0 | + | * Loose refs | 0 | + | * Packed refs | 0 | + | * Refname length | | + | * Local | | + | * Maximum | 0 | + | * Total | 0 | + | * Remote | | + | * Maximum | 0 | + | * Total | 0 | | | | | * Reachable objects | | | * Count | 0 | @@ -76,7 +118,7 @@ test_expect_success 'empty repository' ' test_expect_success SHA1 'repository with references and objects' ' test_when_finished "rm -rf repo" && - git init repo && + git init --initial-branch=main --ref-format=files repo && ( cd repo && test_commit_bulk 1005 && @@ -84,6 +126,8 @@ test_expect_success SHA1 'repository with references and objects' ' oid="$(git rev-parse HEAD)" && git update-ref refs/remotes/origin/foo "$oid" && + git symbolic-ref refs/remotes/origin/HEAD \ + refs/remotes/origin/foo && # Also creates a commit, tree, and blob. git notes add -m foo && @@ -95,12 +139,22 @@ test_expect_success SHA1 'repository with references and objects' ' | Repository structure | Value | | ------------------------- | ---------- | | * References | | - | * Count | 4 | + | * Count | 5 | | * Branches | 1 | | * Tags | 1 | | * Annotated | 1 | - | * Remotes | 1 | + | * Remotes | 2 | | * Others | 1 | + | * Symbolic refs | 1 | + | * Loose refs | 5 | + | * Packed refs | 0 | + | * Refname length | | + | * Local | | + | * Maximum | 18 | + | * Total | 46 | + | * Remote | | + | * Maximum | 24 | + | * Total | 47 | | | | | * Reachable objects | | | * Count | 3.02 k | @@ -141,14 +195,14 @@ test_expect_success SHA1 'repository with references and objects' ' git repo structure >out 2>err && - test_cmp expect out && + check_structure_summary out && test_line_count = 0 err ) ' test_expect_success SHA1 'lines and nul format' ' test_when_finished "rm -rf repo" && - git init repo && + git init --initial-branch=main --ref-format=files repo && ( cd repo && test_commit_bulk 42 && @@ -161,6 +215,13 @@ test_expect_success SHA1 'lines and nul format' ' references.tags.annotated.count=1 references.remotes.count=0 references.others.count=0 + references.symbolic.count=0 + references.loose.count=3 + references.packed.count=0 + references.local.max_length=53 + references.local.total_length=81 + references.remotes.max_length=0 + references.remotes.total_length=0 objects.commits.count=42 objects.trees.count=42 objects.blobs.count=42 @@ -189,13 +250,13 @@ test_expect_success SHA1 'lines and nul format' ' git repo structure --format=lines >out 2>err && - test_cmp expect out && + check_structure_summary out && test_line_count = 0 err && git repo structure --format=nul >out 2>err && tr "\012\000" "=\012" actual && - test_cmp expect actual && + check_structure_summary actual && test_line_count = 0 err && # "-z", as a synonym to "--format=nul", participates in the @@ -203,7 +264,7 @@ test_expect_success SHA1 'lines and nul format' ' git repo structure --format=table -z >out 2>err && tr "\012\000" "=\012" actual && - test_cmp expect actual && + check_structure_summary actual && test_line_count = 0 err ) ' @@ -228,6 +289,839 @@ test_expect_success 'progress meter option' ' ) ' +test_expect_success 'object histograms cover size and entry boundaries' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + for size in 0 1 15 16 255 256 4095 4096 + do + test-tool genzeros "$size" >blob-$size && + git add blob-$size || return 1 + done && + tree=$(git write-tree) && + commit=$(git commit-tree "$tree" -m blobs) && + git update-ref refs/heads/blobs "$commit" && + empty_blob=$(git hash-object -w --stdin tree-input && + tree=$(git mktree expect-unsorted && + sort expect-unsorted >expect-sizes && + git repo structure --format=lines >out && + sed -n "/^objects\..*\.histogram\.size\./p" \ + out >actual-unsorted && + sort actual-unsorted >actual-sizes && + test_cmp expect-sizes actual-sizes && + + cat >expect <<-\EOF && + objects.blobs.histogram.size.0.count=3 + objects.blobs.histogram.size.1.count=2 + objects.blobs.histogram.size.2.count=2 + objects.blobs.histogram.size.3.count=1 + EOF + sed -n "/^objects\.blobs\.histogram\..*\.count=/p" \ + out >actual && + test_cmp expect actual && + + key=objects.trees.histogram.entries && + cat >expect <<-EOF && + $key.0.count=2 + $key.1.count=3 + $key.2.count=2 + $key.3.count=1 + EOF + sed -n "/^$key\..*\.count=/p" out >actual && + test_cmp expect actual && + + git repo structure --format=nul >nul && + tr "\012\000" "=\012" decoded && + test_cmp out decoded && + + git repo structure >table && + for kind in "Commit parent" "Commit size" \ + "Tree entry" "Tree size" "Blob size" + do + test_grep "| $kind histogram " table || + return 1 + done && + sed -n "/^| Blob size histogram /,\$p" table >blobs && + for range in 0..15 16..255 256..4095 4096..65535 + do + test_grep -F "| * $range " blobs || return 1 + done && + + tree=$(git rev-parse refs/heads/tree-16^{tree}) && + format="%(objectsize) %(objectsize:disk)" && + echo "$tree" | + git cat-file --batch-check="$format" \ + >tree-size && + read inflated disk expect <<-EOF && + $key.2.count=1 + $key.2.inflated_size=$inflated + $key.2.disk_size=$disk + EOF + git repo structure --format=lines \ + --ref-filter=refs/heads/tree-16 >filtered && + sed -n "/^objects\.trees\.histogram\.entries\./p" \ + filtered >actual && + test_cmp expect actual || return 1 + done && + + git repo structure --format=lines \ + --ref-filter=refs/heads/missing >out && + test_grep ! "\.histogram\." out + ) +' + +test_expect_success 'commit parent histogram groups 31 or more parents' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + tree=$(git mktree expect <<-\EOF && + objects.commits.histogram.parents.0.count=32 + objects.commits.histogram.parents.1.count=1 + objects.commits.histogram.parents.2.count=1 + objects.commits.histogram.parents.31.count=2 + EOF + git repo structure --format=lines >out && + sed -n "/^objects\.commits\.histogram\.parents\./p" \ + out >actual && + test_cmp expect actual && + test_grep "^objects.commits.max_parents=32$" out && + git repo structure >table && + test_grep "^| 31+ *| *2 *|$" table && + + key=objects.commits.largest.by_parents && + boundary=$(git rev-parse refs/heads/boundary) && + cat >expect <<-EOF && + $key.1.parents=32 + $key.1.oid=$commit + $key.1.commit_oid=$commit + $key.1.name_rev=overflow + $key.2.parents=31 + $key.2.oid=$boundary + $key.2.commit_oid=$boundary + $key.2.name_rev=boundary + $key.3.parents=2 + $key.3.oid=$two + $key.3.commit_oid=$two + $key.3.name_rev=two + EOF + git repo structure --format=lines --commit-parents=3 >out && + sed -n "/^$key\./p" out >actual && + test_cmp expect actual && + + for limit in 9 10 12 + do + git repo structure --format=lines \ + --commit-parents=$limit >out && + sed -n "s/\.oid=/.commit_oid=/p" out >expect && + sed -n "/\.commit_oid=/p" out >actual && + test_line_count = $limit actual && + test_cmp expect actual && + sed -n "s/^.*\.commit_oid=//p" out >oids && + git name-rev --name-only --annotate-stdin \ + expect && + sed -n "s/^.*\.name_rev=//p" out >actual && + test_cmp expect actual && + git repo structure --commit-parents=$limit >table && + sed -n "/^| Largest commits by parent count /,/^$/p" \ + table >parents && + test_line_count = $((limit + 3)) parents && + awk " + !NF { next } + !width { width = length } + length != width { + print length, width + exit 1 + } + " parents || return 1 + done + ) +' + +test_expect_success 'largest object lists have independent sorted limits' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + empty_blob=$(git hash-object -w --stdin big && + big=$(git hash-object -w big) && + printf abcdefghijklmnop >tied && + tied=$(git hash-object -w tied) && + empty_tree=$(git mktree tree-input <<-EOF && + 100644 blob $empty_blob a + 100644 blob $big b + 100644 blob $tied c + EOF + wide=$(git mktree tree-input <<-EOF && + 100644 blob $empty_blob $long-a + 100644 blob $big $long-b + EOF + narrow=$(git mktree message && + root=$(git commit-tree "$empty_tree" -F message) && + other=$(git commit-tree "$narrow" -m other) && + child=$(git commit-tree "$wide" -p "$root" -m child) && + merge=$(git commit-tree "$wide" -p "$child" -p "$other" \ + -m merge) && + git update-ref refs/heads/main "$merge" && + git update-ref refs/heads/empty "$root" && + root_size=$(git cat-file -s "$root") && + narrow_size=$(git cat-file -s "$narrow") && + commit_parents=objects.commits.largest.by_parents && + commit_sizes=objects.commits.largest.by_size && + tree_entries=objects.trees.largest.by_entries && + tree_sizes=objects.trees.largest.by_size && + blob_sizes=objects.blobs.largest.by_size && + printf "%s %s\n" \ + "$big" b "$big" "$long-b" \ + "$tied" c \ + "$empty_blob" a "$empty_blob" "$long-a" \ + >allowed-paths && + + set -- --commit-parents=2 --commit-sizes=1 \ + --tree-entries=4 --tree-sizes=1 --blob-sizes=4 && + for storage in loose packed + do + if test "$storage" = packed + then + git repack -ad + fi && + git repo structure --format=lines "$@" >out && + cat >expect <<-EOF && + $commit_parents.1.parents=2 + $commit_parents.1.oid=$merge + $commit_parents.1.commit_oid=$merge + $commit_parents.1.name_rev=main + $commit_parents.2.parents=1 + $commit_parents.2.oid=$child + $commit_parents.2.commit_oid=$child + $commit_parents.2.name_rev=main~1 + $commit_sizes.1.inflated_size=$root_size + $commit_sizes.1.oid=$root + $commit_sizes.1.commit_oid=$root + $commit_sizes.1.name_rev=empty + $tree_entries.1.entries=3 + $tree_entries.1.oid=$wide + $tree_entries.1.path= + $tree_entries.2.entries=2 + $tree_entries.2.oid=$narrow + $tree_entries.2.path= + $tree_entries.3.entries=0 + $tree_entries.3.oid=$empty_tree + $tree_entries.3.path= + $tree_sizes.1.inflated_size=$narrow_size + $tree_sizes.1.oid=$narrow + $tree_sizes.1.path= + EOF + sed -n "/^$blob_sizes\./d; /\.largest\./p" \ + out >actual && + test_cmp expect actual && + cat >expect <<-EOF && + $blob_sizes.1.inflated_size=16 + $blob_sizes.2.inflated_size=16 + $blob_sizes.3.inflated_size=0 + EOF + sed -n "/^$blob_sizes\..*\.inflated_size=/p" \ + out >actual && + test_cmp expect actual && + printf "%s\n" "$big" "$tied" >expect-unsorted && + sort expect-unsorted >expect && + sed -n "s/^$blob_sizes\.[12]\.oid=//p" \ + out >actual-unsorted && + sort actual-unsorted >actual && + test_cmp expect actual && + test_grep "^$blob_sizes.3.oid=$empty_blob$" out && + test_grep ! "^$blob_sizes.4." out && + for rank in 1 2 3 + do + oid=$(sed -n \ + "s/^$blob_sizes.$rank.oid=//p" out) && + path=$(sed -n \ + "s/^$blob_sizes.$rank.path=//p" out) && + test_grep -F -x "$oid $path" allowed-paths || + return 1 + done && + + git repo structure --format=nul "$@" >nul && + tr "\012\000" "=\012" decoded && + test_cmp out decoded && + git repo structure "$@" >table && + sed -n "/^| Largest commits by parent count /,/^$/p" \ + table >parents && + test_grep "(commit $merge) (main) *\[1\] | *2 *|$" \ + parents && + sed -n "/^| Largest blobs by size /,/^$/p" \ + table >blobs && + test_grep "^| 1: .* \[1\] | *16 B *|$" blobs || + return 1 + done && + + git repo structure --format=lines "$@" \ + --ref-filter=refs/heads/empty >out && + test_grep "^$commit_parents.1.parents=0$" out && + test_grep ! "^$commit_parents.2." out && + test_grep "^$tree_entries.1.entries=0$" out && + test_grep ! "^$tree_entries.2." out && + test_grep ! "^$blob_sizes." out && + git repo structure --format=lines "$@" \ + --ref-filter=refs/heads/missing >out && + test_grep ! "\.largest\." out + ) +' + +test_expect_success 'largest object paths stay with their ranked objects' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + mkdir dir && + i=0 && + >objects && + for size in 1 9 2 8 3 7 + do + i=$((i + 1)) && + test-tool genzeros "$size" >"dir/$i" && + oid=$(git hash-object -w "dir/$i") && + printf "%s %s %s\n" "$size" "$oid" "dir/$i" \ + >>objects || return 1 + done && + git add dir && + test_tick && + git commit -m paths && + dir=$(git rev-parse HEAD:dir) && + dir_size=$(git cat-file -s "$dir") && + entries=objects.trees.largest.by_entries && + sizes=objects.trees.largest.by_size && + blobs=objects.blobs.largest.by_size && + sort -rn objects >sorted && + for limit in 1 2 4 + do + cat >expect <<-EOF && + $entries.1.entries=6 + $entries.1.oid=$dir + $entries.1.path=dir/ + $sizes.1.inflated_size=$dir_size + $sizes.1.oid=$dir + $sizes.1.path=dir/ + EOF + sed -n "1,${limit}p" sorted >selected && + rank=0 && + while read -r size oid path + do + rank=$((rank + 1)) && + key=$blobs.$rank && + printf "%s=%s\n" \ + "$key.inflated_size" "$size" \ + "$key.oid" "$oid" \ + "$key.path" "$path" >>expect || + return 1 + done out && + sed -n "/\.largest\./p" out >actual && + test_cmp expect actual || return 1 + done + ) +' + +test_expect_success 'largest object paths are quoted except in NUL output' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test-tool genzeros 16 >blob && + blob=$(git hash-object -w blob) && + printf "100644 blob %s\tfile\0" "$blob" >tree-input && + dir=$(git mktree -z tree-input && + root=$(git mktree -z expect <<-\EOF && + objects.trees.largest.by_size.1.path= + objects.trees.largest.by_size.2.path="d\t\n\"\\x/" + objects.blobs.largest.by_size.1.path="d\t\n\"\\x/file" + EOF + git repo structure --format=lines "$@" >out && + sed -n "/\.largest\..*\.path=/p" out >actual && + test_cmp expect actual && + + cat >patterns <<-\EOF && + 2: "d\t\n\"\\x/" + 1: "d\t\n\"\\x/file" + EOF + git repo structure "$@" >table && + while IFS= read -r pattern + do + test_grep -F "$pattern" table || return 1 + done expect && + printf "%s\n%s\0" \ + "$trees.1.inflated_size" "$root_size" \ + "$trees.1.oid" "$root" \ + "$trees.1.path" "" \ + "$trees.2.inflated_size" "$dir_size" \ + "$trees.2.oid" "$dir" \ + "$trees.2.path" "$name/" \ + "$blobs.1.inflated_size" 16 \ + "$blobs.1.oid" "$blob" \ + "$blobs.1.path" "$name/file" >>expect && + git repo structure --format=nul "$@" >actual && + test_cmp expect actual + ) +' + +test_expect_success 'ranked revision names are batched and use all refs' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test_commit --no-tag one file && + git tag -a -m tag v1 && + name=tags/v1^0 && + key=objects.commits.largest && + set -- --commit-parents=4 --commit-sizes=4 \ + --tree-entries=4 --tree-sizes=4 --blob-sizes=4 && + GIT_TRACE2_EVENT="$PWD/trace" git repo structure \ + --format=lines --progress \ + --ref-filter=refs/heads/main "$@" >out 2>err && + printf "%s=%s\n" \ + "$key.by_parents.1.name_rev" "$name" \ + "$key.by_size.1.name_rev" "$name" >expect && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect actual && + test_grep "^references.tags.count=0$" out && + test_grep "Resolving revision names" err && + grep "child_start.*\"name-rev\"" trace >children && + test_line_count = 1 children && + + GIT_TRACE2_EVENT="$PWD/default-trace" \ + git repo structure >out && + test_grep ! "child_start.*\"name-rev\"" default-trace && + GIT_TRACE2_EVENT="$PWD/tree-trace" git repo structure \ + --tree-entries=4 --tree-sizes=4 --blob-sizes=4 >out && + test_grep ! "child_start.*\"name-rev\"" tree-trace + ) +' + +test_expect_success 'revision-name lookup can be disabled by option or config' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test_commit --no-tag one file && + oid=$(git rev-parse HEAD) && + name_rev_report () { + git "$@" --commit-parents=2 --commit-sizes=2 \ + --tree-entries=2 --tree-sizes=2 --blob-sizes=2 + } && + name_rev_report repo structure --format=lines >enabled && + test_grep "\.name_rev=" enabled && + sed "/\.name_rev=/d" enabled >disabled && + child_pattern="child_start.*\"name-rev\"" && + for spec in \ + "default enabled" \ + "default enabled --name-rev" \ + "default disabled --no-name-rev" \ + "true enabled" \ + "false disabled" \ + "bare enabled" \ + "true disabled --no-name-rev" \ + "false enabled --name-rev" \ + "true disabled --name-rev --no-name-rev" \ + "false enabled --no-name-rev --name-rev" + do + set -- $spec && + config=$1 expected=$2 && + shift 2 && + set -- repo structure "$@" && + case "$config" in + default) ;; + bare) set -- -c repo.structure.nameRev "$@" ;; + *) set -- -c repo.structure.nameRev=$config "$@" ;; + esac && + >trace && + test_env GIT_TRACE2_EVENT="$PWD/trace" \ + git "$@" --format=lines --commit-parents=2 \ + --commit-sizes=2 --tree-entries=2 \ + --tree-sizes=2 --blob-sizes=2 >actual 2>err && + test_cmp "$expected" actual && + test_must_be_empty err && + case "$expected" in + enabled) + grep "$child_pattern" trace >children && + test_line_count = 1 children + ;; + disabled) + test_grep ! "\"name-rev\"" trace + ;; + esac || return 1 + done && + + name_rev_report repo structure --no-name-rev \ + --format=nul >nul && + tr "\012\000" "=\012" actual && + test_cmp disabled actual && + name_rev_report repo structure --no-name-rev \ + --progress >table 2>err && + test_grep -F "(commit $oid)" table && + test_grep ! -F "(main)" table && + test_grep ! "Resolving revision names" err && + + test_must_fail git -c repo.structure.nameRev=invalid \ + repo structure --commit-parents=1 2>err && + test_grep "bad boolean config value" err + ) +' + +test_expect_success 'ranked revision names quote text but preserve NUL data' ' + test_when_finished "rm -rf repo" && + name=$(printf "q\042\303\251") && + git init --ref-format=reftable --initial-branch="$name" repo && + ( + cd repo && + test_commit --no-tag one file && + oid=$(git rev-parse HEAD) && + key=objects.commits.largest.by_parents && + cat >expect <<-\EOF && + objects.commits.largest.by_parents.1.name_rev="q\"\303\251" + EOF + git -c core.quotePath=true repo structure \ + --commit-parents=2 --format=lines >out && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect actual && + cat >pattern <<-\EOF && + ("q\"\303\251") + EOF + git -c core.quotePath=true repo structure \ + --commit-parents=2 >table && + grep -F -f pattern table >found && + test_line_count = 1 found && + + printf "%s=\042q\134\042\303\251\042\n" \ + "$key.1.name_rev" >expect && + git -c core.quotePath=false repo structure \ + --commit-parents=2 --format=lines >out && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect actual && + + git repo structure --format=nul >expect && + printf "%s\n%s\0" \ + "$key.1.parents" 0 \ + "$key.1.oid" "$oid" \ + "$key.1.commit_oid" "$oid" \ + "$key.1.name_rev" "$name" >>expect && + git repo structure --commit-parents=2 --format=nul >actual && + test_cmp expect actual + ) +' + +test_expect_success 'revision-name failures leave other statistics intact' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test_commit --no-tag one file && + oid=$(git rev-parse HEAD) && + set -- --commit-parents=3 --commit-sizes=3 --format=lines && + git repo structure "$@" >original && + sed "/\.name_rev=/d" original >expect && + printf "%s\n" "$oid" "$oid" >expect-input && + mkdir mock && + write_script mock/git <<-\EOF && + if test "$*" != "name-rev --name-only --annotate-stdin" + then + echo "unexpected command: $*" >&2 + exit 1 + fi + cat >name-rev-input || exit 1 + case "$NAME_REV_MODE" in + fail) exit 1 ;; + short) printf "first\n" ;; + extra) printf "first\nsecond\nthird\n" ;; + unterminated) printf "first\nsecond" ;; + nul) printf "first\0ignored\nsecond\n" ;; + empty) printf "first\n\n" ;; + raw) cat name-rev-input ;; + crlf) printf "first\r\nsecond\r\n" ;; + esac + EOF + for mode in fail short extra unterminated nul empty + do + NAME_REV_MODE=$mode git --exec-path="$PWD/mock" \ + repo structure "$@" >out 2>err && + test_cmp expect-input name-rev-input && + test_cmp expect out && + case "$mode" in + fail) + test_grep "could not resolve revision names" err + ;; + *) + test_grep "unexpected output.*name-rev" err + ;; + esac || return 1 + done && + + key=objects.commits.largest && + for mode in raw crlf + do + NAME_REV_MODE=$mode git --exec-path="$PWD/mock" \ + repo structure "$@" >out 2>err && + test_must_be_empty err && + test_cmp expect-input name-rev-input && + sed "/\.name_rev=/d" out >actual && + test_cmp expect actual && + case "$mode" in + raw) first=$oid second=$oid ;; + crlf) first=first second=second ;; + esac && + printf "%s=%s\n" \ + "$key.by_parents.1.name_rev" "$first" \ + "$key.by_size.1.name_rev" "$second" \ + >expect-names && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect-names actual || return 1 + done + ) +' + +for spec in \ + "commit-parents showCommitParents commits by_parents parents" \ + "commit-sizes showCommitSizes commits by_size inflated_size" \ + "tree-entries showTreeEntries trees by_entries entries" \ + "tree-sizes showTreeSizes trees by_size inflated_size" \ + "blob-sizes showBlobSizes blobs by_size inflated_size" +do + set -- $spec + option=$1 config=$2 type=$3 dimension=$4 metric=$5 + + test_expect_success "--$option is opt-in and overrides its config" ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test_commit --no-tag one file && + case "$type" in + commits) oid=$(git rev-parse HEAD) ;; + trees) oid=$(git rev-parse HEAD^{tree}) ;; + blobs) oid=$(git rev-parse HEAD:file) ;; + esac && + case "$metric" in + parents) value=0 ;; + entries) value=1 ;; + inflated_size) value=$(git cat-file -s "$oid") ;; + esac && + key=objects.$type.largest.$dimension && + cat >expect <<-EOF && + $key.1.$metric=$value + $key.1.oid=$oid + EOF + case "$type" in + commits) + printf "%s=%s\n" "$key.1.commit_oid" "$oid" \ + "$key.1.name_rev" main >>expect + ;; + trees) echo "$key.1.path=" >>expect ;; + blobs) echo "$key.1.path=file" >>expect ;; + esac && + git repo structure --format=lines >default && + test_grep ! "\.largest\." default && + git repo structure --format=lines "--$option=3" >out && + sed -n "/\.largest\./p" out >actual && + test_cmp expect actual && + git repo structure --format=nul "--$option=3" >nul && + tr "\012\000" "=\012" decoded && + test_cmp out decoded && + git repo structure "--$option=3" >table && + case "$type" in + commits) test_grep -F "(commit $oid) (main)" table ;; + *) test_grep ! -F "(commit " table ;; + esac && + git -c repo.structure.$config=3 repo structure \ + --format=lines >configured && + test_cmp out configured && + git -c repo.structure.$config=3 repo structure \ + --format=lines "--$option=0" >disabled && + test_cmp default disabled && + git -c repo.structure.$config=0 repo structure \ + --format=lines "--$option=3" >override && + test_cmp out override && + + test_commit --no-tag two other "larger content" && + git -c repo.structure.$config=1 repo structure \ + --format=lines >limit-one && + git -c repo.structure.$config=2 repo structure \ + --format=lines >limit-two && + git repo structure --format=lines "--$option=2" \ + >expected-two && + test_cmp expected-two limit-two && + sed -n "/\.largest\..*\.oid=/p" limit-one >oids && + test_line_count = 1 oids && + sed -n "/\.largest\..*\.oid=/p" limit-two >oids && + test_line_count = 2 oids && + + test_must_fail git repo structure "--$option=-1" \ + 2>err && + test_grep "must be non-negative" err && + test_must_fail git -c repo.structure.$config=-1 \ + repo structure 2>err && + test_grep "must be non-negative" err && + test_must_fail git repo structure "--$option=bad" \ + 2>err + ) + ' +done + +test_expect_success 'largest object lists are empty in an empty repository' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git repo structure >expect && + git repo structure --commit-parents=3 --commit-sizes=3 \ + --tree-entries=3 --tree-sizes=3 --blob-sizes=3 \ + >actual && + test_cmp expect actual + ) +' + +for ref_format in files reftable +do + test_expect_success "$ref_format reference statistics" ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main \ + --ref-format="$ref_format" repo && + ( + cd repo && + test_commit --no-tag one && + git tag v1 && + git update-ref refs/notes/commits HEAD && + git update-ref refs/remotes/origin/long-branch HEAD && + git symbolic-ref refs/heads/alias refs/heads/main && + git symbolic-ref refs/remotes/origin/HEAD \ + refs/remotes/origin/long-branch && + git pack-refs --all && + test_commit --no-tag two && + + if test "$ref_format" = files + then + loose=3 && + packed=3 + else + loose=0 && + packed=0 + fi && + cat >expect <<-EOF && + references.branches.count=2 + references.tags.count=1 + references.tags.annotated.count=0 + references.remotes.count=2 + references.others.count=1 + references.symbolic.count=2 + references.loose.count=$loose + references.packed.count=$packed + references.local.max_length=18 + references.local.total_length=61 + references.remotes.max_length=31 + references.remotes.total_length=55 + EOF + git repo structure --format=lines >out && + sed -n "/^references\./p" out >actual && + test_cmp expect actual && + + if test "$ref_format" = files + then + loose=1 && + packed=1 + fi && + cat >expect <<-EOF && + references.branches.count=0 + references.tags.count=0 + references.tags.annotated.count=0 + references.remotes.count=2 + references.others.count=0 + references.symbolic.count=1 + references.loose.count=$loose + references.packed.count=$packed + references.local.max_length=0 + references.local.total_length=0 + references.remotes.max_length=31 + references.remotes.total_length=55 + EOF + git repo structure --format=nul \ + --ref-filter="refs/remotes/origin/*" >out && + tr "\012\000" "=\012" decoded && + sed -n "/^references\./p" decoded >actual && + test_cmp expect actual && + + sed "s/=[0-9]*$/=0/" expect >expect-empty && + git repo structure --format=lines \ + --ref-filter="refs/does-not-exist/" >out && + sed -n "/^references\./p" out >actual && + test_cmp expect-empty actual + ) + ' +done + test_expect_success '--ref-filter narrows the set of refs' ' test_when_finished "rm -rf repo" && git init repo && diff --git a/t/t4001-diff-rename.sh b/t/t4001-diff-rename.sh index ad474100affa0b..1c56ee25651f58 100755 --- a/t/t4001-diff-rename.sh +++ b/t/t4001-diff-rename.sh @@ -126,6 +126,21 @@ test_expect_success 'test diff.renames unset' ' compare_diff_patch current expected ' +test_expect_success 'diff.renameThreshold=100% suppresses inexact rename in diff' ' + git -c diff.renameThreshold=100% diff --cached $tree >current && + compare_diff_patch current no-rename +' + +test_expect_success 'diff.renameThreshold=1% detects rename in diff' ' + git -c diff.renameThreshold=1% diff --cached $tree >current && + compare_diff_patch current expected +' + +test_expect_success '-M overrides diff.renameThreshold' ' + git -c diff.renameThreshold=100% diff -M --cached $tree >current && + compare_diff_patch current expected +' + test_expect_success 'favour same basenames over different ones' ' cp path1 another-path && git add another-path && @@ -155,6 +170,16 @@ test_expect_success 'favour same basenames even with minor differences' ' test_grep "renamed: .*path1 -> subdir/path1" out ' +test_expect_success 'diff.renameThreshold with modified rename in status' ' + git show HEAD:path1 | sed -e "s/Line 1/Changed 1/" \ + -e "s/Line 2/Changed 2/" -e "s/Line 3/Changed 3/" >subdir/path1 && + git add subdir/path1 && + git -c diff.renameThreshold=100% status >out && + test_grep ! "renamed:" out && + git -c diff.renameThreshold=1% status >out && + test_grep "renamed:" out +' + test_expect_success 'two files with same basename and same content' ' git reset --hard && mkdir -p dir/A dir/B && diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index aac139e6a096eb..26f5c3d78d6c35 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -381,6 +381,30 @@ test_expect_success 'build pack index for an existing pack' ' : ' +# The `--rev-index` option of `git index-pack` is now the default, so +# a `foo.rev` REV file will be created when a `foo.idx` IDX file is +# created. Normally, these pathnames are based upon the `foo.pack` +# PACK file pathname. +# +# However, the `-o` option lets you set the pathname of the IDX file +# indepdent of the PACK file. +# +# Verify what happens if these suffixes are changed. +# +test_expect_success 'complain about index name' ' + # Normal case { .pack, .idx, .rev } + cat test-1-${packname_1}.pack >test-complain-0.pack && + git index-pack -o test-complain-0.idx --rev-index test-complain-0.pack && + test -f test-complain-0.idx && + test -f test-complain-0.rev && + + # Non .idx suffix -- implicitly omits the .rev + cat test-1-${packname_1}.pack >test-complain-1.pack && + git index-pack -o test-complain-1.idx-suffix --rev-index test-complain-1.pack && + test -f test-complain-1.idx-suffix && + ! test -f test-complain-1.rev +' + test_expect_success 'unpacking with --strict' ' for j in a b c d e f g diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index c660a487e36801..0c3043da2954f9 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -1418,18 +1418,21 @@ test_expect_success 'lookup recovers object whose midx-owning pack was removed' git add dup && git commit -m dup && dup_oid=$(git rev-parse HEAD:dup) && + dup_commit=$(git rev-parse HEAD) && # Roll every object, including dup, into a single big pack. git repack -adq && # Build a second, "moderate" pack that also contains dup, so dup # now lives in two packs that the midx will cover. - moderate=$(echo "$dup_oid" | + moderate=$(printf "%s\n" "$dup_oid" "$dup_commit" | git pack-objects --quiet $objdir/pack/pack) && # Attribute dup to the moderate pack in the midx. git multi-pack-index write \ --preferred-pack="pack-$moderate.idx" && + test_commit child && + git rev-list HEAD >expect-commits && # Simulate a concurrent "git repack" retiring the moderate pack: # its files disappear, but the now-stale midx still names it as @@ -1441,7 +1444,13 @@ test_expect_success 'lookup recovers object whose midx-owning pack was removed' # would appear missing even though it is physically present. echo blob >expect && git cat-file -t "$dup_oid" >actual && - test_cmp expect actual + test_cmp expect actual && + GIT_TRACE2_EVENT="$PWD/no-fetch.trace" \ + git -c core.gvfs=0 -c core.commitGraph=false \ + -c core.multiPackIndex=true -c core.useGVFSHelper=true \ + rev-list --missing=print HEAD >actual && + test_cmp expect-commits actual && + test_grep ! child_start no-fetch.trace ) ' diff --git a/t/t5590-push-path-walk.sh b/t/t5590-push-path-walk.sh new file mode 100755 index 00000000000000..7849ec337b0aa7 --- /dev/null +++ b/t/t5590-push-path-walk.sh @@ -0,0 +1,109 @@ +#!/bin/sh + +test_description='verify that push respects `pack.usePathWalk`' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-pack.sh + +test_expect_success 'setup bare repository and clone' ' + git init --bare -b main bare.git && + git --git-dir=bare.git config receive.unpackLimit 0 && + git --git-dir bare.git commit-tree -m initial $EMPTY_TREE >head_oid && + git --git-dir bare.git update-ref refs/heads/main $(cat head_oid) && + git clone --bare bare.git clone.git +' +test_expect_success 'avoid reusing deltified objects' ' + # construct two commits, one containing a file with the hex digits + # repeated 16 times, the next reducing that to 8 times. The crucial + # part is that the blob of the second commit is deltified _really_ + # badly and it is therefore easy to detect if a `git push` reused that + # delta. + x="0123456789abcdef" && + printf "$x$x$x$x$x$x$x$x" >x128 && + printf "$x$x$x$x$x$x$x$x$x$x$x$x$x$x$x$x" >x256 && + + pack=clone.git/objects/pack/pack-tmp.pack && + pack_header 2 >$pack && + + # add x256 as a non-deltified object, using an uncompressed zlib stream + # for simplicity + # 060 = OBJ_BLOB << 4, 0200 = size larger than 15, + # 0 = lower 4 bits of size, 020 = bits 5-9 of size (size = 256) + printf "\260\020" >>$pack && + # Uncompressed zlib stream always starts with 0170 1 1, followed + # by two bytes encoding the size, little endian, then two bytes with + # the bitwise-complement of that size, then the payload, and then the + # Adler32 checksum. For some reason, the checksum is in big-endian + # format. + printf "\170\001\001\0\001\377\376" >>$pack && + cat x256 >>$pack && + # Manually-computed Adler32 checksum: 0xd7ae4621 + printf "\327\256\106\041" >>$pack && + + # add x128 as a very badly deltified object + # 0120 = OBJ_OFS_DELTA << 4, 0200 = total size larger than 15, + # 4 = lower 4 bits of size, 030 = bits 5-9 of size + # (size = 128 * 3 + 2 + 2) + printf "\344\030" >>$pack && + # 0415 = size (i.e. the relative negative offset) of the previous + # object (x256, used as base object) + # encoded as 0200 | ((0415 >> 7) - 1), 0415 & 0177 + printf "\201\015" >>$pack && + # Uncompressed zlib stream, as before, size = 2 + 2 + 128 * 3 (i.e. + # 0604) + printf "\170\001\001\204\001\173\376" >>$pack && + # base object size = 0400 (encoded as 0200 | (0400 & 0177), + # 0400 >> 7) + printf "\200\002" >>$pack && + # object size = 0200 (encoded as 0200 | (0200 & 0177), 0200 >> 7 + printf "\200\001" >>$pack && + # massively badly-deltified object: copy every single byte individually + # 0200 = copy, 1 = use 1 byte to encode the offset (counter), + # 020 = use 1 byte to encode the size (1) + printf "$(printf "\\\\221\\\\%03o\\\\001" $(test_seq 0 127))" >>$pack && + # Manually-computed Adler32 checksum: 0x99c369c4 + printf "\231\303\151\304" >>$pack && + + pack_trailer $pack && + git index-pack -v $pack && + + oid256=$(git hash-object x256) && + printf "100755 blob $oid256\thex\n" >tree && + tree_oid="$(git --git-dir=clone.git mktree tree && + tree_oid="$(git --git-dir=clone.git mktree verify && + size="$(sed -n "s/^$oid128 blob *\([^ ]*\).*/\1/p" verify && + size="$(sed -n "s/^$oid128 blob *\([^ ]*\).*/\1/p" a && + git add a && + git commit -m initial && + git clone . one +' + +test_expect_success "fetch test" ' + cd one && + git config core.gvfs 16 && + rm -rf .git/objects/* && + git -C .. cat-file commit HEAD | git hash-object -w --stdin -t commit && + git fetch && + test_must_fail git rev-parse --verify HEAD^{tree} +' + +test_done diff --git a/t/t5613-info-alternate.sh b/t/t5613-info-alternate.sh index c752804a8e90cc..44ffe4688ab351 100755 --- a/t/t5613-info-alternate.sh +++ b/t/t5613-info-alternate.sh @@ -137,4 +137,16 @@ test_expect_success CASE_INSENSITIVE_FS 'dup finding can be case-insensitive' ' test_cmp expect actual.alternates ' +test_expect_success 'unusable alternates only warn on config lookup' ' + git init unusable && + printf "%s\n" ../missing ../missing-parent/objects \ + >unusable/.git/objects/info/alternates && + test_expect_code 1 git -C unusable config --get test.missing \ + >actual 2>err && + test_must_be_empty actual && + test_line_count = 2 err && + test_grep "^warning: object directory" err && + test_grep "^warning: unable to normalize alternate object path" err +' + test_done diff --git a/t/t5615-alternate-env.sh b/t/t5615-alternate-env.sh index 1bfeccdeb49958..d82dba65d94b14 100755 --- a/t/t5615-alternate-env.sh +++ b/t/t5615-alternate-env.sh @@ -88,4 +88,47 @@ test_expect_success !MINGW 'broken quoting falls back to interpreting raw' ' EOF ' +test_expect_success 'packs across sources are checked before loose objects' ' + # Regression test for a performance issue in which an object that + # resides in an alternate as a packed object caused a spurious loose + # object lookup (a filesystem stat) on the main object store before the + # alternate packfile was consulted. Reading such an object must resolve + # to the alternate packfile, never to a loose copy in the main store. + # + # Build an alternate whose object "B" is stored as a delta in a + # packfile. git deltifies successive versions of a tracked file, so the + # older, shorter blob "B" becomes a delta against the newer, longer + # blob "O". A loose object has no delta base, so %(deltabase) tells us + # which store answered the read: the alternate pack (O) or a loose copy + # (the zero oid). + git init alt-src && + test_seq 1 200 >alt-src/file && + git -C alt-src add file && + git -C alt-src commit -q -m base && + B=$(git -C alt-src rev-parse HEAD:file) && + git -C alt-src cat-file blob "$B" >b-content && + test_seq 1 210 >alt-src/file && + git -C alt-src add file && + git -C alt-src commit -q -m more && + O=$(git -C alt-src rev-parse HEAD:file) && + git -C alt-src repack -adf --window=10 --depth=50 && + + # Precondition: in the alternate, B is a delta based on O. + echo "$B" >in && + echo "$O" >expect && + git -C alt-src cat-file --batch-check="%(deltabase)" actual && + test_cmp expect actual && + + # Main repo: write B as a loose object, before any alternate is active. + git init main && + git -C main hash-object -w --stdin /dev/null && + test -e "main/.git/objects/$(test_oid_to_path "$B")" && + + # With the alternate active, B must resolve to the alternate packfile + # (deltabase O), not to the main store loose copy (deltabase zero oid). + GIT_ALTERNATE_OBJECT_DIRECTORIES="$PWD/alt-src/.git/objects" \ + git -C main cat-file --batch-check="%(deltabase)" actual && + test_cmp expect actual +' + test_done diff --git a/t/t5790-gvfs-helper-basic.sh b/t/t5790-gvfs-helper-basic.sh new file mode 100755 index 00000000000000..093c809dcccaa1 --- /dev/null +++ b/t/t5790-gvfs-helper-basic.sh @@ -0,0 +1,338 @@ +#!/bin/sh + +test_description='gvfs-helper basic tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Basic tests to confirm the happy path works. +################################################################# + +test_expect_success 'basic: GET origin multi-get no-auth' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OIDS_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OIDS_FILE" && + verify_connection_count 1 +' + +test_expect_success 'basic: GET cache-server multi-get trust-mode' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the cache-server and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OIDS_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OIDS_FILE" && + verify_connection_count 1 +' + +test_expect_success 'basic: GET gvfs/config' ' +# test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the cache-server and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + config \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # The cache-server URL should be listed in the gvfs/config output. + # We confirm this before assuming error-mode will work. + # + test_grep "$CACHE_URL" OUT.output +' + +test_expect_success 'basic: GET cache-server multi-get error-mode' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the cache-server and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=error \ + --remote=origin \ + get \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OIDS_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OIDS_FILE" && + + # Technically, we have 1 connection to the origin server + # for the "gvfs/config" request and 1 to cache server to + # get the objects, but because we are using the same port + # for both, keep-alive will handle it. So 1 connection. + # + verify_connection_count 1 +' + +# The GVFS Protocol POST verb behaves like GET for non-commit objects +# (in that it just returns the requested object), but for commit +# objects POST *also* returns all trees referenced by the commit. +# +# The goal of this test is to confirm that gvfs-helper can send us +# a packfile at all. So, this test only passes blobs to not blur +# the issue. +# +test_expect_success 'basic: POST origin blobs' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and make + # multi-object POST request. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "packfile " message for each received + # packfile. We verify the number of expected packfile(s) and we + # individually verify that each requested object is present in the + # shared cache (and index-pack already verified the integrity of + # the packfile), so we do not bother to run "git verify-pack -v" + # and do an exact matchup here. + # + verify_received_packfile_count 1 && + + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + verify_connection_count 1 +' + +# Request a single blob via POST. Per the GVFS Protocol, the server +# should implicitly send a loose object for it. Confirm that. +# +test_expect_success 'basic: POST-request a single blob' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and request a single + # blob via POST. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received + # loose object. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_connection_count 1 +' + +# Request a single commit via POST. Per the GVFS Protocol, the server +# should implicitly send us a packfile containing the commit and the +# trees it references. Confirm that properly handled the receipt of +# the packfile. (Here, we are testing that asking for a single commit +# via POST yields a packfile rather than a loose object.) +# +# We DO NOT verify that the packfile contains commits/trees and no blobs +# because our test helper doesn't implement the filtering. +# +test_expect_success 'basic: POST-request a single commit' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and request a single + # commit via POST. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OID_ONE_COMMIT_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 1 && + + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH w/o arg gets all' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Without a "since" argument gives us all "ct-*.pack" since the EPOCH + # because we do not have any prefetch packs locally. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 3 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH w/ arg' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Ask for cached packfiles NEWER THAN the given time. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH mayhem no_prefetch_idx' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem no_prefetch_idx && + + # Request prefetch packs, but tell server to not send any + # idx files and force gvfs-helper to compute them. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH up-to-date' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Ask for cached packfiles NEWER THAN the given time. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + # Ask again for any packfiles newer than what we have cached locally. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 0 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 2 +' + +test_done diff --git a/t/t5791-gvfs-helper-errors.sh b/t/t5791-gvfs-helper-errors.sh new file mode 100755 index 00000000000000..189950c097253b --- /dev/null +++ b/t/t5791-gvfs-helper-errors.sh @@ -0,0 +1,353 @@ +#!/bin/sh + +test_description='gvfs-helper error handling tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Tests to see how gvfs-helper responds to network problems. +# +# We use small --max-retry value because of exponential backoff. +# +# These mayhem tests are interested in how gvfs-helper gracefully +# retries when there is a network error. And verify that it gives +# up gracefully too. +################################################################# + +mayhem_observed__close__connections () { + if grep "transient" OUT.stderr + then + # Transient errors should retry. + # 1 for initial request + 2 retries. + # + verify_connection_count 3 + return $? + elif grep "hard_fail" OUT.stderr + then + # Hard errors should not retry. + # + verify_connection_count 1 + return $? + else + error "mayhem_observed__close: unexpected mayhem-induced error type" + return 1 + fi +} + +mayhem_observed__close () { + # Expected error codes for mayhem events: + # close_read + # close_write + # close_no_write + # + # CURLE_PARTIAL_FILE 18 + # CURLE_GOT_NOTHING 52 + # CURLE_SEND_ERROR 55 + # CURLE_RECV_ERROR 56 + # + # I don't want to pin it down to an exact error for each because there may + # be races here because of network buffering. + # + # Also, It is unclear which of these network errors should be transient + # (with retry) and which should be a hard-fail (without retry). I'm only + # going to verify the connection counts based upon what type of error + # gvfs-helper claimed it to be. + # + if grep "error: get: (curl:18)" OUT.stderr || + grep "error: get: (curl:52)" OUT.stderr || + grep "error: get: (curl:55)" OUT.stderr || + grep "error: get: (curl:56)" OUT.stderr + then + mayhem_observed__close__connections + return $? + else + echo "mayhem_observed__close: unexpected mayhem-induced error" + return 1 + fi +} + +test_lazy_prereq CURL_8_16_0 ' + git gvfs-helper curl-version = 8.16.0 || + test 8.15.0-DEV = "$(git gvfs-helper curl-version)" +' + +test_expect_success 'curl-error: no server' ' + test_when_finished "per_test_cleanup" && + + connect_timeout_ms= && + # CURLE_COULDNT_CONNECT 7 + regex="error: get: (curl:7)" && + if test_have_prereq CURL_8_16_0 + then + connect_timeout_ms=--connect-timeout-ms=200 && + # CURLE_COULDNT_CONNECT 7 + # CURLE_OPERATION_TIMEDOUT 28 + regex="error: get: (curl:\(7\|28\))" + fi && + + # Try to do a multi-get without a server. + # + # Use small max-retry value because of exponential backoff, + # but yet do exercise retry some. + # + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + $connect_timeout_ms \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + test_grep "$regex" OUT.stderr +' + +test_expect_success 'curl-error: close socket while reading request' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_read && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + mayhem_observed__close +' + +test_expect_success 'curl-error: close socket while writing response' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_write && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + mayhem_observed__close +' + +test_expect_success 'curl-error: close socket before writing response' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_no_write && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + mayhem_observed__close +' + +################################################################# +# Tests to confirm that gvfs-helper does silently recover when +# a retry succeeds. +# +# Note: I'm only to do this for 1 of the close_* mayhem events. +################################################################# + +test_expect_success 'successful retry after curl-error: origin get' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_read_1 && + + # Connect to the origin server (w/o auth). + # Make a single-object GET request. + # Confirm that it succeeds without error. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + verify_connection_count 2 +' + +################################################################# +# Tests to see how gvfs-helper responds to HTTP errors/problems. +# +################################################################# + +# See "enum gh__error_code" in gvfs-helper.c +# +GH__ERROR_CODE__HTTP_404=4 +GH__ERROR_CODE__HTTP_429=5 +GH__ERROR_CODE__HTTP_503=6 + +test_expect_success 'http-error: 503 Service Unavailable (with retry)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_503 && + + test_expect_code $GH__ERROR_CODE__HTTP_503 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:503)" OUT.stderr && + verify_connection_count 3 +' + +test_expect_success 'http-error: 429 Service Unavailable (with retry)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_429 && + + test_expect_code $GH__ERROR_CODE__HTTP_429 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:429)" OUT.stderr && + verify_connection_count 3 +' + +test_expect_success 'http-error: 404 Not Found (no retry)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_404 && + + test_expect_code $GH__ERROR_CODE__HTTP_404 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:404)" OUT.stderr && + verify_connection_count 1 +' + +################################################################# +# Tests to confirm that gvfs-helper does silently recover when an +# HTTP request succeeds after a failure. +# +################################################################# + +test_expect_success 'successful retry after http-error: origin get' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_429_1 && + + # Connect to the origin server (w/o auth). + # Make a single-object GET request. + # Confirm that it succeeds without error. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + verify_connection_count 2 +' + +################################################################# +# So far we have confirmed that gvfs-helper can recover from a network +# error (with retries, since the cache-server was disabled in all of +# the above tests). Try again with fallback turned on. +# +# With mayhem "http_503" turned on both the cache and origin server +# will always throw a 503 error. +# +# Confirm that we tried to make six connections: we should hit the +# cache-server 3 times (one initial attempt and two retries) and then +# try the origin server 3 times. +# +################################################################# + +test_expect_success 'http-error: 503 Service Unavailable (with retry and fallback)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_503 && + + test_expect_code $GH__ERROR_CODE__HTTP_503 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --fallback \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:503)" OUT.stderr && + verify_connection_count 6 +' + +################################################################# +# Now repeat the above, but explicitly turn off fallback. +# +# Again, we use mayhem "http_503". However, with fallback turned +# off, we will only attempt the 3 connections to the cache server. +# We will not try to hit the origin server. +# +# So we should only see a total of 3 connections rather than the +# six in the previous test. +# +################################################################# + +test_expect_success 'http-error: 503 Service Unavailable (with retry and no-fallback)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_503 && + + test_expect_code $GH__ERROR_CODE__HTTP_503 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-fallback \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:503)" OUT.stderr && + verify_connection_count 3 +' + +test_done diff --git a/t/t5792-gvfs-helper-auth.sh b/t/t5792-gvfs-helper-auth.sh new file mode 100755 index 00000000000000..0aa9188664652f --- /dev/null +++ b/t/t5792-gvfs-helper-auth.sh @@ -0,0 +1,111 @@ +#!/bin/sh + +test_description='gvfs-helper authentication tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Test HTTP Auth +# +################################################################# + +test_lazy_prereq CURL_7_75_OR_NEWER ' + git gvfs-helper curl-version ">=" 7.75.0 +' + +test_expect_success 'HTTP GET Auth on Origin Server' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_401 && + + # Force server to require auth. + # Connect to the origin server without auth. + # Make a single-object GET request. + # Confirm that it gets a 401 and then retries with auth. + # + GIT_CONFIG_NOSYSTEM=1 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + if test_have_prereq CURL_7_75_OR_NEWER + then + verify_connection_count 2 + fi +' + +test_expect_success 'HTTP POST Auth on Origin Server' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_401 && + + # Connect to the origin server and make multi-object POST + # request and verify that it automatically handles the 401. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "packfile " message for each received + # packfile. We verify the number of expected packfile(s) and we + # individually verify that each requested object is present in the + # shared cache (and index-pack already verified the integrity of + # the packfile), so we do not bother to run "git verify-pack -v" + # and do an exact matchup here. + # + verify_received_packfile_count 1 && + + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + verify_connection_count 2 +' + +test_expect_success 'HTTP GET Auth on Cache Server' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_401 && + + # Try auth to cache-server. Note that gvfs-helper *ALWAYS* sends + # creds to cache-servers, so we will never see the "400 Bad Request" + # response. And we are using "trust" mode, so we only expect 1 + # connection to the server. + # + GIT_CONFIG_NOSYSTEM=1 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + verify_connection_count 1 +' + +test_done diff --git a/t/t5793-gvfs-helper-integration.sh b/t/t5793-gvfs-helper-integration.sh new file mode 100755 index 00000000000000..dced714f5b7301 --- /dev/null +++ b/t/t5793-gvfs-helper-integration.sh @@ -0,0 +1,216 @@ +#!/bin/sh + +test_description='gvfs-helper integration tests with Git commands' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Integration tests with Git.exe +# +# Now that we have confirmed that gvfs-helper works in isolation, +# run a series of tests using random Git commands that fault-in +# objects as needed. +# +# At this point, I'm going to stop verifying the shape of the ODB +# (loose vs packfiles) and the number of connections required to +# get them. The tests from here on are to verify that objects are +# magically fetched whenever required. +################################################################# + +test_expect_success 'integration: explicit commit/trees, implicit blobs: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # We have a very empty repo. Seed it with all of the commits + # and trees. The purpose of this test is to demand-load the + # needed blobs only, so we prefetch the commits and trees. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + <"$OIDS_CT_FILE" >OUT.output && + + # Confirm that we do not have the blobs locally. + # With gvfs-helper turned off, we should fail. + # + test_must_fail \ + git -C "$REPO_T1" -c core.useGVFSHelper=false \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + # Turn on gvfs-helper and retry. This should implicitly fetch + # any needed blobs. + # + git -C "$REPO_T1" -c core.useGVFSHelper=true \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + # Verify that gvfs-helper wrote the fetched the blobs to the + # local ODB, such that a second attempt with gvfs-helper + # turned off should succeed. + # + git -C "$REPO_T1" -c core.useGVFSHelper=false \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr +' + +trace_has_queue_oid () { + oid=$1 + grep "gh_client__queue_oid: $oid" +} + +trace_has_immediate_oid () { + oid=$1 + grep "gh_client__get_immediate: $oid" +} + +test_expect_success 'integration: fully implicit: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Implicitly demand-load everything without any pre-seeding. + # + GIT_TRACE2_EVENT="$(pwd)/diff-trace.txt" \ + git -C "$REPO_T1" -c core.useGVFSHelper=true \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + oid=$(git -C "$REPO_SRC" rev-parse main:file9.txt.t) && + trace_has_queue_oid $oid OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +test_expect_success 'integration: implicit-get: cache_http_503,no-fallback: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + + # Tell cache server to send 503 and origin server to send 200. + start_gvfs_protocol_server_with_mayhem cache_http_503 && + + # Implicitly demand-load everything without any pre-seeding. + # This should fail because we do not allow fallback. + # + test_must_fail \ + git -C "$REPO_T2" \ + -c core.useGVFSHelper=true \ + -c gvfs.fallback=false \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +test_expect_success 'integration: implicit-get: cache_http_503,with-fallback: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + + # Tell cache server to send 503 and origin server to send 200. + start_gvfs_protocol_server_with_mayhem cache_http_503 && + + # Implicitly demand-load everything without any pre-seeding. + # + git -C "$REPO_T2" \ + -c core.useGVFSHelper=true \ + -c gvfs.fallback=true \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +# T2 should be considered contaminated at this point. + +################################################################# +# Test X-Session-Id header +# +# The X-Session-Id header should contain the SID (session ID). +################################################################# + +test_expect_success 'integration: X-Session-Id header with and without prefix' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Case 1: No gvfs.sessionkey configured - should send just SID + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output1 && + + # Verify X-Session-Id contains SID (with process ID marker "-P") + test_grep "X-Session-Id:.*-P" "$SERVER_LOG" >OUT.case1 && + # Verify no slash (no prefix) + test_grep ! "X-Session-Id:.*:" OUT.case1 && + + # Case 2: gvfs.sessionkey points to non-existent config - should send just SID + rm -f OUT.output* OUT.case* && + git -C "$REPO_T1" -c gvfs.sessionkey="test.id" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output2 && + + # Verify X-Session-Id still contains just SID (no prefix) + test_grep "X-Session-Id:.*-P" "$SERVER_LOG" >OUT.case2 && + test_grep ! "X-Session-Id:.*:" OUT.case2 && + + # Case 3: gvfs.sessionkey points to existing config - should send prefix/SID + rm -f OUT.output* OUT.case* && + git -C "$REPO_T1" \ + -c gvfs.sessionkey="test.id" \ + -c test.id="my-trace-12345" \ + gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output3 && + + # Verify X-Session-Id contains prefix, slash, and SID + test_grep "X-Session-Id:.*my-trace-12345:" "$SERVER_LOG" >OUT.case3 && + test_grep "X-Session-Id:.*my-trace-12345:.*-P" OUT.case3 +' + +test_done diff --git a/t/t5794-gvfs-helper-packfiles.sh b/t/t5794-gvfs-helper-packfiles.sh new file mode 100755 index 00000000000000..6e9fd1510d5a74 --- /dev/null +++ b/t/t5794-gvfs-helper-packfiles.sh @@ -0,0 +1,197 @@ +#!/bin/sh + +test_description='gvfs-helper packfile handling tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Duplicate packfile tests. +# +# If we request a fixed set of blobs, we should get a unique packfile +# of the form "vfs-.{pack,idx}". It we request that same set +# again, the server should create and send the exact same packfile. +# True web servers might build the custom packfile in random order, +# but our test web server should give us consistent results. +# +# Verify that we can handle the duplicate pack and idx file properly. +################################################################# + +first_received_packfile_pathname () { + sed -n "s/packfile //p" OUT.output 2>OUT.stderr && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + # Re-fetch the same packfile. We do not care if it replaces + # first one or if it silently fails to overwrite the existing + # one. We just confirm that afterwards we only have 1 packfile. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + stop_gvfs_protocol_server +' + +test_expect_success 'duplicate and busy: vfs- packfile' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" \ + >OUT.output \ + 2>OUT.stderr && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + # Re-fetch the same packfile, but hold the existing packfile + # open for writing on an obscure (and randomly-chosen) file + # descriptor. + # + # This should cause the replacement-install to fail (at least + # on Windows) with an EBUSY or EPERM or something. + # + # Verify that that error is eaten. We do not care if the + # replacement is retried or if gvfs-helper simply discards the + # second instance. We just confirm that afterwards we only + # have 1 packfile on disk and that the command "lies" and reports + # that it created the existing packfile. (We want the lie because + # in normal usage, gh-client has already built the packed-git list + # in memory and is using gvfs-helper to fetch missing objects; + # gh-client does not care who does the fetch, but it needs to + # update its packed-git list and restart the object lookup.) + # + PACK=$(first_received_packfile_pathname) && + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" \ + >OUT.output \ + 2>OUT.stderr \ + 9>>"$PACK" && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + stop_gvfs_protocol_server +' + +################################################################# +# Ensure that the SHA of the blob we received matches the SHA of +# the blob we requested. +################################################################# + +# Request a loose blob from the server. Verify that we received +# content matches the requested SHA. +# +test_expect_success 'catch corrupted loose object' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem corrupt_loose && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + # Verify corruption detected. + # Verify valid blob not included in response to client. + + test_grep "hash failed for received loose object" OUT.stderr && + + # Verify that we did not write the corrupted blob to the ODB. + + ! verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + git -C "$REPO_T1" fsck +' + +################################################################# +# Ensure that we can detect when we receive a corrupted packfile +# from the server. This is not concerned with network IO errors, +# but rather cases when the cache or origin server generates or +# sends an invalid packfile. +# +# For example, if the server throws an exception and writes the +# stack trace to the socket rather than or in addition to the +# packfile content. +# +# Or for example, if the packfile on the server's disk is corrupt +# and it sends it correctly, but the original data was already +# garbage, so the client still has garbage (and retrying won't +# help). +################################################################# + +# Send corrupt PACK files w/o IDX files (so that `gvfs-helper` +# must use `index-pack` to create it. (And as a side-effect, +# validate the PACK file is not corrupt.) +test_expect_success 'prefetch corrupt pack without idx' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem \ + bad_prefetch_pack_sha \ + no_prefetch_idx && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch \ + --max-retries=0 \ + --since="1000000000" \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + # Verify corruption detected in pack when building + # local idx file for it. + + test_grep "error: .* index-pack failed" OUT.stderr +' + +# Send corrupt PACK files with IDX files. Since the cache server +# sends both, `gvfs-helper` might fail to verify both of them. +test_expect_success 'prefetch corrupt pack with corrupt idx' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem \ + bad_prefetch_pack_sha && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch \ + --max-retries=0 \ + --since="1000000000" \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +test_done diff --git a/t/t5795-gvfs-helper-verb-cache.sh b/t/t5795-gvfs-helper-verb-cache.sh new file mode 100755 index 00000000000000..653deeedd5554d --- /dev/null +++ b/t/t5795-gvfs-helper-verb-cache.sh @@ -0,0 +1,224 @@ +#!/bin/sh + +test_description='gvfs-helper verb-specific cache-server tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Tests for gvfs..cache-server config. +# +# These tests verify that verb-specific cache-server overrides work +# correctly. We run two servers on different ports: +# - Server 0 (base port): configured as gvfs.cache-server (default) +# - Server 1 (base port + 1): configured as gvfs..cache-server +# +# For each verb (prefetch, get, post), we verify that: +# 1. When using the verb-specific override, the request goes to server 1 +# 2. When using a different verb, the request goes to server 0 +################################################################# + +test_expect_success 'verb-specific cache-server: prefetch uses gvfs.prefetch.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for prefetch. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + + # Run prefetch - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (prefetch-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: get does NOT use gvfs.prefetch.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for prefetch. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 0 (default), not server 1 (prefetch). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (prefetch-specific). + verify_server_was_not_contacted 1 +' + +test_expect_success 'verb-specific cache-server: get uses gvfs.get.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for get. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (get-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: prefetch does NOT use gvfs.get.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for get. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && + + # Run prefetch - should go to server 0 (default), not server 1 (get). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (get-specific). + verify_server_was_not_contacted 1 +' + +test_expect_success 'verb-specific cache-server: post uses gvfs.post.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for post. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && + + # Run post - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (post-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: get does NOT use gvfs.post.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for post. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 0 (default), not server 1 (post). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (post-specific). + verify_server_was_not_contacted 1 +' + +test_expect_success 'verb-specific cache-server: all verbs with different servers' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && + start_gvfs_protocol_server 3 && + + # Configure each verb to use a different server: + # - server 0: default (unused in this test) + # - server 1: prefetch + # - server 2: get + # - server 3: post + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 2)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 3)" && + + # Run prefetch - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + verify_server_was_contacted 1 && + verify_server_was_not_contacted 0 && + verify_server_was_not_contacted 2 && + verify_server_was_not_contacted 3 && + + # Clean up shared cache for next verb. + rm -rf "$SHARED_CACHE_T1"/pack/* && + + # Run get - should go to server 2. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + verify_server_was_contacted 2 && + + # Clean up shared cache for next verb. + rm -rf "$SHARED_CACHE_T1"/[0-9a-f][0-9a-f]/ && + rm -rf "$SHARED_CACHE_T1"/pack/* && + + # Run post - should go to server 3. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + verify_server_was_contacted 3 +' + +test_done diff --git a/t/t5797-gvfs-helper-prefetch-threads.sh b/t/t5797-gvfs-helper-prefetch-threads.sh new file mode 100755 index 00000000000000..a215570ccfada6 --- /dev/null +++ b/t/t5797-gvfs-helper-prefetch-threads.sh @@ -0,0 +1,153 @@ +#!/bin/sh + +test_description='gvfs-helper prefetch with gvfs.prefetchThreads config + +Verify that the prefetch verb works correctly in both sequential +(gvfs.prefetchThreads=1) and parallel (gvfs.prefetchThreads=4) modes. +Each test is run under both configurations to ensure identical results +and to exercise both code paths in install_prefetch(). +' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +# Helper: run a prefetch that fetches all 3 epoch packs (no --since). +# +do_prefetch_all () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_received_packfile_count 3 && + verify_prefetch_keeps 1200000000 +} + +# Helper: run a prefetch with --since to get 2 of 3 packs. +# +do_prefetch_since () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 +} + +# Helper: prefetch then re-prefetch to verify up-to-date handling. +# +do_prefetch_up_to_date () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + # Re-fetch; should find nothing new. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_received_packfile_count 0 && + verify_prefetch_keeps 1200000000 +} + +# Helper: prefetch corrupt pack (error path). +# Requires the server to be started with the appropriate mayhem. +# +do_prefetch_corrupt_pack () { + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch \ + --max-retries=0 \ + --since="1000000000" \ + >OUT.output 2>OUT.stderr && + + test_grep "error: .* index-pack failed" OUT.stderr +} + +for threads in 1 4 +do + # Describe the mode for readable test names. + if test "$threads" = "1" + then + mode="sequential" + # The sequential path logs install_mode=1. + expected_mode=1 + else + mode="parallel" + expected_mode=$threads + fi + + test_expect_success "prefetch all packs ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_prefetch_all && + + stop_gvfs_protocol_server && + + test_trace2_data gvfs-helper prefetch/install_mode '$expected_mode' \ + <"trace-$test_count.txt" + ' + + test_expect_success "prefetch with --since ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_prefetch_since && + + stop_gvfs_protocol_server && + + test_trace2_data gvfs-helper prefetch/install_mode '$expected_mode' \ + <"trace-$test_count.txt" + ' + + test_expect_success "prefetch up-to-date ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + do_prefetch_up_to_date && + + stop_gvfs_protocol_server + ' + + test_expect_success "prefetch corrupt pack ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem \ + bad_prefetch_pack_sha \ + no_prefetch_idx && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + do_prefetch_corrupt_pack && + + stop_gvfs_protocol_server + ' +done + +test_done diff --git a/t/t5798-gvfs-helper-post-threads.sh b/t/t5798-gvfs-helper-post-threads.sh new file mode 100755 index 00000000000000..4f5998d073f810 --- /dev/null +++ b/t/t5798-gvfs-helper-post-threads.sh @@ -0,0 +1,469 @@ +#!/bin/sh + +test_description='gvfs-helper POST with gvfs.postThreads config + +Verify that the post verb works correctly in both sequential +(gvfs.postThreads=1) and parallel (gvfs.postThreads=4) modes. +Each test is run under both configurations to ensure identical results +and to exercise both code paths in do__http_post__fetch_oidset(). +' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +parallel_block_size=100 + +test_expect_success 'create enough blobs for parallel POST' ' + test_commit_bulk -C "$REPO_SRC" --filename="parallel.%s.t" 101 && + get_list_of_blobs_oids +' + +# Helper: POST a set of OIDs and verify we get the expected packfiles. +# +do_post_blobs () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_received_packfile_count 1 && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" +} + +# Helper: POST blobs with the minimum parallel block size. +# +do_post_blobs_small_blocks () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + --block-size="$parallel_block_size" \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" +} + +# Helper: leave one OID after the first nominal block. The parallel +# partitioner must avoid sending that object as a loose-object response to +# index-pack. +# +do_post_blobs_single_oid_remainder () { + nr_oids=$(sort -u "$OIDS_BLOBS_FILE" | wc -l) && + block_size=$(($nr_oids - 1)) && + + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + --block-size="$block_size" \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" +} + +# Helper: POST same set twice to test duplicate handling. +# +do_post_duplicate () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_received_packfile_count 1 && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + + # Second fetch of same objects should still succeed. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" +} + +verify_parallel_post_workers () { + trace_file=$1 && + + test_trace2_data gvfs-helper post/fetch_mode 4 <"$trace_file" && + nr_workers=$(grep "\"key\":\"post/worker\"" "$trace_file" | + sed -n "s/.*\"value\":\"\\([0-9]*\\)\".*/\\1/p" | + sort -u | wc -l) && + test "$nr_workers" -gt 1 +} + +do_post_corrupt_pack () { + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + --block-size="$parallel_block_size" \ + --max-retries=0 \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_grep "error: post: index-pack failed" OUT.stderr +} + +for value in unset 0 negative +do + test_expect_success "postThreads=$value uses sequential mode" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + if test "'$value'" = unset + then + git -C "$REPO_T1" config --unset-all \ + gvfs.postThreads || : + elif test "'$value'" = negative + then + git -C "$REPO_T1" config gvfs.postThreads -1 + else + git -C "$REPO_T1" config gvfs.postThreads 0 + fi && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_blobs_small_blocks && + + stop_gvfs_protocol_server && + test_trace2_data gvfs-helper post/fetch_mode 1 \ + <"trace-$test_count.txt" + ' +done + +test_expect_success 'malformed postThreads is rejected' ' + test_when_finished "git -C \"$REPO_T1\" config --unset-all \ + gvfs.postThreads" && + git -C "$REPO_T1" config gvfs.postThreads invalid && + + test_must_fail git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + test_grep "bad numeric config value" OUT.stderr +' + +test_expect_success PTHREADS 'small block size uses sequential POST' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + --block-size=$(($parallel_block_size - 1)) \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + stop_gvfs_protocol_server && + test_trace2_data gvfs-helper post/fetch_mode 1 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'cookie configuration uses sequential POST' ' + test_when_finished "per_test_cleanup" && + test_when_finished "rm -f cookies" && + >"cookies" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + git -C "$REPO_T1" \ + -c http.cookieFile="$(pwd)/cookies" \ + -c http.saveCookies=true \ + gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + --block-size="$parallel_block_size" \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + stop_gvfs_protocol_server && + test_trace2_data gvfs-helper post/fetch_mode 1 \ + <"trace-$test_count.txt" +' + +for threads in 1 4 +do + if test "$threads" = "1" + then + mode="sequential" + prereq= + expected_mode=1 + else + mode="parallel" + prereq=PTHREADS + expected_mode=$threads + fi + + test_expect_success "$prereq" \ + "post blobs ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_blobs && + + stop_gvfs_protocol_server && + + test_trace2_data gvfs-helper post/fetch_mode '$expected_mode' \ + <"trace-$test_count.txt" + ' + + test_expect_success "$prereq" \ + "post small blocks ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_blobs_small_blocks && + + stop_gvfs_protocol_server && + + if test '$threads' = 4 + then + verify_parallel_post_workers \ + "trace-$test_count.txt" + else + test_trace2_data gvfs-helper post/fetch_mode 1 \ + <"trace-$test_count.txt" + fi + ' + + test_expect_success "$prereq" \ + "post single-OID remainder ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_blobs_single_oid_remainder && + + stop_gvfs_protocol_server && + + test_trace2_data gvfs-helper post/fetch_mode '$expected_mode' \ + <"trace-$test_count.txt" + ' + + test_expect_success "$prereq" \ + "post duplicate ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_duplicate && + + stop_gvfs_protocol_server && + + test_trace2_data gvfs-helper post/fetch_mode '$expected_mode' \ + <"trace-$test_count.txt" + ' +done + +test_expect_success PTHREADS,PERL_TEST_HELPERS 'parallel POST does not deadlock' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + "$PERL_PATH" -e "alarm shift; exec @ARGV or die \$!" -- 30 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + --block-size="$parallel_block_size" \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + stop_gvfs_protocol_server && + test_trace2_data gvfs-helper post/fetch_mode 4 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'parallel POST reports index-pack failure' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem bad_post_pack_sha && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_corrupt_pack && + + stop_gvfs_protocol_server && + test_grep "bad_post_pack_sha" "$SERVER_LOG" && + test_trace2_data gvfs-helper post/fetch_mode 4 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'parallel POST retries a corrupt pack' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem bad_post_pack_sha_1 && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_blobs_small_blocks && + + stop_gvfs_protocol_server && + test_grep "bad_post_pack_sha_1" "$SERVER_LOG" && + test_trace2_data gvfs-helper post/fetch_mode 4 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'parallel POST retries a transient HTTP error' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_429_1 && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_blobs_small_blocks && + + stop_gvfs_protocol_server && + test_grep "http_429_1" "$SERVER_LOG" && + test_trace2_data gvfs-helper post/fetch_mode 4 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'parallel POST retries authentication' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_401_1 && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_post_blobs_small_blocks && + + stop_gvfs_protocol_server && + test_grep "http_401_1" "$SERVER_LOG" && + test_trace2_data gvfs-helper post/fetch_mode 4 \ + <"trace-$test_count.txt" && + test_trace2_data gvfs-helper post/auth_retry 1 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'parallel POST falls back after cache 404' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem cache_http_404 && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --fallback \ + --no-progress \ + post \ + --block-size="$parallel_block_size" \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + stop_gvfs_protocol_server && + test_grep "cache_http_404" "$SERVER_LOG" && + test_trace2_data gvfs-helper post/fetch_mode 4 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'parallel POST honors --no-fallback' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem cache_http_404 && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-fallback \ + --no-progress \ + post \ + --block-size="$parallel_block_size" \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_grep "error: post: (http:404)" OUT.stderr && + stop_gvfs_protocol_server && + test_grep "cache_http_404" "$SERVER_LOG" && + test_trace2_data gvfs-helper post/fetch_mode 4 \ + <"trace-$test_count.txt" +' + +test_expect_success PTHREADS 'parallel POST preserves configured headers' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.postThreads 4 && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + git -C "$REPO_T1" \ + -c http.extraHeader="X-Test-Header: parallel" \ + -c gvfs.sessionkey=test.id \ + -c test.id=parallel-session \ + gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + --block-size="$parallel_block_size" \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + test_must_be_empty OUT.stderr && + stop_gvfs_protocol_server && + test_grep "X-Test-Header: parallel" "$SERVER_LOG" && + test_grep "X-Session-Id:.*parallel-session:.*-P" "$SERVER_LOG" && + verify_parallel_post_workers "trace-$test_count.txt" +' + +test_done diff --git a/t/t6434-merge-recursive-rename-options.sh b/t/t6434-merge-recursive-rename-options.sh index 5a6f74839cb716..00ae0080c06041 100755 --- a/t/t6434-merge-recursive-rename-options.sh +++ b/t/t6434-merge-recursive-rename-options.sh @@ -332,4 +332,36 @@ test_expect_success 'merge.renames overrides diff.renames' ' $check_50 ' +test_expect_success 'diff.renameThreshold sets merge threshold' ' + git read-tree --reset -u HEAD && + test_must_fail git -c diff.renameThreshold=$th0 merge-recursive $tail && + check_threshold_0 +' + +test_expect_success 'diff.renameThreshold=100% limits to exact renames in merge' ' + git read-tree --reset -u HEAD && + test_must_fail git -c diff.renameThreshold=100% merge-recursive $tail && + check_exact_renames +' + +test_expect_success 'merge.renameThreshold overrides diff.renameThreshold' ' + git read-tree --reset -u HEAD && + test_must_fail git -c diff.renameThreshold=100% \ + -c merge.renameThreshold=$th0 merge-recursive $tail && + check_threshold_0 +' + +test_expect_success 'merge.renameThreshold defaults to diff.renameThreshold' ' + git read-tree --reset -u HEAD && + test_must_fail git -c diff.renameThreshold=$th2 merge-recursive $tail && + check_threshold_2 +' + +test_expect_success '--find-renames overrides merge.renameThreshold' ' + git read-tree --reset -u HEAD && + test_must_fail git -c merge.renameThreshold=100% \ + merge-recursive --find-renames=$th0 $tail && + check_threshold_0 +' + test_done diff --git a/t/t7002-mv-sparse-checkout.sh b/t/t7002-mv-sparse-checkout.sh index 9c0e82ba3190a7..537948c5850d85 100755 --- a/t/t7002-mv-sparse-checkout.sh +++ b/t/t7002-mv-sparse-checkout.sh @@ -155,6 +155,9 @@ test_expect_success 'mv refuses to move sparse-to-non-sparse' ' test_expect_success 'recursive mv refuses to move (possible) sparse' ' test_when_finished rm -rf b c e sub2 && + + git config advice.sparseIndexExpanded false && + git reset --hard && # Without cone mode, "sub" and "sub2" do not match git sparse-checkout set sub/dir sub2/dir && diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh index b7cbcbf869296c..db5483b8f10052 100755 --- a/t/t7108-reset-stdin.sh +++ b/t/t7108-reset-stdin.sh @@ -29,4 +29,13 @@ test_expect_success '--stdin requires --mixed' ' git reset --mixed --stdin list && + git reset --stdin marker + EOF + + : make sure -changed is called if -change does not exist && + test_when_finished "echo testing >dir1/file2.txt && git status" && + echo changed >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_file marker && + + test_when_finished "rm -f .git/hooks/post-index-change marker2" && + write_script .git/hooks/post-index-change <<-\EOF && + : >marker2 + EOF + + : make sure -changed is not called if -change exists && + rm -f marker marker2 && + echo testing >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_missing marker && + test_path_is_file marker2 +' + test_expect_success 'test status, add, commit, others trigger hook without flags set' ' test_hook post-index-change <<-\EOF && if test "$1" -eq 1; then diff --git a/t/t7502-commit-porcelain.sh b/t/t7502-commit-porcelain.sh index dbacc73e9d3537..dd85be52b9e14c 100755 --- a/t/t7502-commit-porcelain.sh +++ b/t/t7502-commit-porcelain.sh @@ -1026,6 +1026,11 @@ EOF ' test_expect_success WITH_BREAKING_CHANGES 'core.commentChar=auto is rejected' ' + cat >&2 <<-EOF && + Trying to run any pre-command hook already triggers a failure when + running \`git config core.commentChar auto\`; Skipping this test. + EOF + return 0 && test_config core.commentChar auto && test_must_fail git rev-parse --git-dir 2>err && sed -n "s/^hint: *\$//p; s/^hint: //p; s/^fatal: //p" err >actual && diff --git a/t/t7519/fsmonitor-watchman b/t/t7519/fsmonitor-watchman index bcc055c1e0945c..9bddc61d863160 100755 --- a/t/t7519/fsmonitor-watchman +++ b/t/t7519/fsmonitor-watchman @@ -17,7 +17,6 @@ use IPC::Open2; # 'git config core.fsmonitor .git/hooks/query-watchman' # my ($version, $time) = @ARGV; -#print STDERR "$0 $version $time\n"; # Check the hook interface version @@ -42,7 +41,7 @@ launch_watchman(); sub launch_watchman { - my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') or die "open2() failed: $!\n" . "Falling back to scanning...\n"; @@ -60,19 +59,11 @@ sub launch_watchman { "fields": ["name"] }] END - - open (my $fh, ">", ".git/watchman-query.json"); - print $fh $query; - close $fh; print CHLD_IN $query; close CHLD_IN; my $response = do {local $/; }; - open ($fh, ">", ".git/watchman-response.json"); - print $fh $response; - close $fh; - die "Watchman: command returned no output.\n" . "Falling back to scanning...\n" if $response eq ""; die "Watchman: command returned invalid output: $response\n" . @@ -91,7 +82,6 @@ sub launch_watchman { my $o = $json_pkg->new->utf8->decode($response); if ($o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { - print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; qx/watchman watch "$git_work_tree"/; die "Failed to make watchman watch '$git_work_tree'.\n" . "Falling back to scanning...\n" if $? != 0; @@ -100,11 +90,6 @@ sub launch_watchman { # return the fast "everything is dirty" flag to git and do the # Watchman query just to get it over with now so we won't pay # the cost in git to look up each individual file. - - open ($fh, ">", ".git/watchman-output.out"); - print "/\0"; - close $fh; - print "/\0"; exit 0; } @@ -112,11 +97,6 @@ sub launch_watchman { die "Watchman: $o->{error}.\n" . "Falling back to scanning...\n" if $o->{error}; - open ($fh, ">", ".git/watchman-output.out"); - binmode $fh, ":utf8"; - print $fh @{$o->{files}}; - close $fh; - binmode STDOUT, ":utf8"; local $, = "\0"; print @{$o->{files}}; diff --git a/t/t7519/fsmonitor-watchman-debug b/t/t7519/fsmonitor-watchman-debug new file mode 100755 index 00000000000000..d8e7a1e5ba85c0 --- /dev/null +++ b/t/t7519/fsmonitor-watchman-debug @@ -0,0 +1,128 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $time) = @ARGV; +#print STDERR "$0 $version $time\n"; + +# Check the hook interface version + +if ($version == 1) { + # convert nanoseconds to seconds + # subtract one second to make sure watchman will return all changes + $time = int ($time / 1000000000) - 1; +} else { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree; +if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $git_work_tree = Win32::GetCwd(); + $git_work_tree =~ tr/\\/\//; +} else { + require Cwd; + $git_work_tree = Cwd::cwd(); +} + +my $retry = 1; + +launch_watchman(); + +sub launch_watchman { + + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $time but were not transient (ie created after + # $time but no longer exist). + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. + + my $query = <<" END"; + ["query", "$git_work_tree", { + "since": $time, + "fields": ["name"] + }] + END + + open (my $fh, ">", ".git/watchman-query.json"); + print $fh $query; + close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; }; + + open ($fh, ">", ".git/watchman-response.json"); + print $fh $response; + close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + my $json_pkg; + eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; + } or do { + require JSON::PP; + $json_pkg = "JSON::PP"; + }; + + my $o = $json_pkg->new->utf8->decode($response); + + if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { + print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; + $retry--; + qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + + open ($fh, ">", ".git/watchman-output.out"); + print "/\0"; + close $fh; + + print "/\0"; + eval { launch_watchman() }; + exit 0; + } + + die "Watchman: $o->{error}.\n" . + "Falling back to scanning...\n" if $o->{error}; + + open ($fh, ">", ".git/watchman-output.out"); + binmode $fh, ":utf8"; + print $fh @{$o->{files}}; + close $fh; + + binmode STDOUT, ":utf8"; + local $, = "\0"; + print @{$o->{files}}; +} diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh new file mode 100755 index 00000000000000..b5c5906187ab7d --- /dev/null +++ b/t/t7522-serialized-status.sh @@ -0,0 +1,461 @@ +#!/bin/sh + +test_description='git serialized status tests' + +. ./test-lib.sh + +# This file includes tests for serializing / deserializing +# status data. These tests cover two basic features: +# +# [1] Because users can request different types of untracked-file +# and ignored file reporting, the cache data generated by +# serialize must use either the same untracked and ignored +# parameters as the later deserialize invocation; otherwise, +# the deserialize invocation must disregard the cached data +# and run a full scan itself. +# +# To increase the number of cases where the cached status can +# be used, we have added a "--untracked-file=complete" option +# that reports a superset or union of the results from the +# "-u normal" and "-u all". We combine this with a filter in +# deserialize to filter the results. +# +# Ignored file reporting is simpler in that is an all or +# nothing; there are no subsets. +# +# The tests here (in addition to confirming that a cache +# file can be generated and used by a subsequent status +# command) need to test this untracked-file filtering. +# +# [2] ensuring the status calls are using data from the status +# cache as expected. This includes verifying cached data +# is used when appropriate as well as falling back to +# performing a new status scan when the data in the cache +# is insufficient/known stale. + +test_expect_success 'setup' ' + git branch -M main && + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt && + + test_oid_cache <<-EOF + branch_oid sha1:68d4a437ea4c2de65800f48c053d4d543b55c410 + x_base sha1:587be6b4c3f93f93c489c0111bba5596147a26cb + x_ours sha1:b68025345d5301abad4d9ec9166f455243a0d746 + x_theirs sha1:975fbec8256d3e8a3797e7a3611380f27c49f4ac + + branch_oid sha256:6b95e4b1ea911dad213f2020840f5e92d3066cf9e38cf35f79412ec58d409ce4 + x_base sha256:14f5162e2fe3d240d0d37aaab0f90e4af9a7cfa79639f3bab005b5bfb4174d9f + x_ours sha256:3a404ba030a4afa912155c476a48a253d4b3a43d0098431b6d6ca6e554bd78fb + x_theirs sha256:44dc634218adec09e34f37839b3840bad8c6103693e9216626b32d00e093fa35 + EOF +' + +test_expect_success 'verify untracked-files=complete with no conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=complete --ignored=matching --deserialize=serialized_status.dat >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=normal conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output && + test_cmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=all conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=all --ignored=matching --deserialize=serialized_status.dat >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_expect_success 'verify serialized status with non-convertible ignore mode does new scan' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? new_change.txt + ? output + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --ignored --deserialize=serialized_status.dat >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_expect_success 'verify serialized status handles path scopes' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? untracked/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat untracked >output && + test_cmp expect output +' + +test_expect_success 'verify no-ahead-behind and serialized status integration' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-EOF && + # branch.oid $(test_oid branch_oid) + # branch.head alt_branch + # branch.upstream main + # branch.ab +1 -0 + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b alt_branch main --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on alt branch" && + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git -c status.aheadBehind=false status --porcelain=v2 --branch --ahead-behind --deserialize=serialized_status.dat >output && + test_cmp expect output +' + +test_expect_success 'verify new --serialize=path mode' ' + test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && + cat >expect <<-\EOF && + ? expect + ? output.1 + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b serialize_path_branch main --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on serialize_path_branch" && + + git status --porcelain=v2 --serialize=serialized_status.dat >output.1 && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 && + test_cmp expect output.1 && + test_cmp expect output.2 +' + +test_expect_success 'try deserialize-wait feature' ' + test_when_finished "rm -f serialized_status.dat dirt expect.* output.* trace.*" && + + git status --serialize=serialized_status.dat >output.1 && + + # make status cache stale by updating the mtime on the index. confirm that + # deserialize fails when requested. + sleep 1 && + touch .git/index && + test_must_fail git status --deserialize=serialized_status.dat --deserialize-wait=fail && + test_must_fail git -c status.deserializeWait=fail status --deserialize=serialized_status.dat && + + cat >expect.1 <<-\EOF && + ? expect.1 + ? output.1 + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + # refresh the status cache. + git status --porcelain=v2 --serialize=serialized_status.dat >output.1 && + test_cmp expect.1 output.1 && + + # create some dirt. confirm deserialize used the existing status cache. + echo x >dirt && + git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 && + test_cmp output.1 output.2 && + + # make the cache stale and try the timeout feature and wait upto + # 2 tenths of a second. confirm deserialize timed out and rejected + # the status cache and did a normal scan. + + cat >expect.2 <<-\EOF && + ? dirt + ? expect.1 + ? expect.2 + ? output.1 + ? output.2 + ? serialized_status.dat + ? trace.2 + ? untracked/ + ? untracked_1.txt + EOF + + sleep 1 && + touch .git/index && + GIT_TRACE_DESERIALIZE=1 git status --porcelain=v2 --deserialize=serialized_status.dat --deserialize-wait=2 >output.2 2>trace.2 && + test_cmp expect.2 output.2 && + grep "wait polled=2 result=1" trace.2 >trace.2g +' + +test_expect_success 'merge conflicts' ' + + # create a merge conflict. + + git init -b main conflicts && + echo x >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m x && + git -C conflicts branch a && + git -C conflicts branch b && + git -C conflicts checkout a && + echo y >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m a && + git -C conflicts checkout b && + echo z >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m b && + test_must_fail git -C conflicts merge --no-commit a && + + # verify that regular status correctly identifies it + # in each format. + + cat >expect.v2 <observed.v2 && + test_cmp expect.v2 observed.v2 && + + cat >expect.long <..." to mark resolution) + both modified: x.txt + +no changes added to commit (use "git add" and/or "git commit -a") +EOF + git -C conflicts status --long >observed.long && + test_cmp expect.long observed.long && + + cat >expect.short <observed.short && + test_cmp expect.short observed.short && + + # save status data in serialized cache. + + git -C conflicts status --serialize >serialized && + + # make some dirt in the worktree so we can tell whether subsequent + # status commands used the cached data or did a fresh status. + + echo dirt >conflicts/dirt.txt && + + # run status using the cached data. + + git -C conflicts status --long --deserialize=../serialized >observed.long && + test_cmp expect.long observed.long && + + git -C conflicts status --short --deserialize=../serialized >observed.short && + test_cmp expect.short observed.short && + + # currently, the cached data does not have enough information about + # merge conflicts for porcelain V2 format. (And V2 format looks at + # the index to get that data, but the whole point of the serialization + # is to avoid reading the index unnecessarily.) So V2 always rejects + # the cached data when there is an unresolved conflict. + + cat >expect.v2.dirty <observed.v2 && + test_cmp expect.v2.dirty observed.v2 + +' + +test_expect_success 'renames' ' + git init -b main rename_test && + echo OLDNAME >rename_test/OLDNAME && + git -C rename_test add OLDNAME && + git -C rename_test commit -m OLDNAME && + git -C rename_test mv OLDNAME NEWNAME && + git -C rename_test status --serialize=renamed.dat >output.1 && + echo DIRT >rename_test/DIRT && + git -C rename_test status --deserialize=renamed.dat >output.2 && + test_cmp output.1 output.2 +' + +test_expect_success 'hint message when cached with u=complete' ' + git init -b main hint && + echo xxx >hint/xxx && + git -C hint add xxx && + git -C hint commit -m xxx && + + cat >expect.clean <expect.use_u <hint.output_normal && + test_cmp expect.clean hint.output_normal && + + git -C hint status --untracked-files=all >hint.output_all && + test_cmp expect.clean hint.output_all && + + git -C hint status --untracked-files=no >hint.output_no && + test_cmp expect.use_u hint.output_no && + + # Create long format output for "complete" and create status cache. + + git -C hint status --untracked-files=complete --ignored=matching --serialize=../hint.dat >hint.output_complete && + test_cmp expect.clean hint.output_complete && + + # Capture long format output using the status cache and verify + # that the output matches the non-cached version. There are 2 + # ways to specify untracked-files, so do them both. + + git -C hint status --deserialize=../hint.dat -unormal >hint.d1_normal && + test_cmp expect.clean hint.d1_normal && + git -C hint -c status.showuntrackedfiles=normal status --deserialize=../hint.dat >hint.d2_normal && + test_cmp expect.clean hint.d2_normal && + + git -C hint status --deserialize=../hint.dat -uall >hint.d1_all && + test_cmp expect.clean hint.d1_all && + git -C hint -c status.showuntrackedfiles=all status --deserialize=../hint.dat >hint.d2_all && + test_cmp expect.clean hint.d2_all && + + git -C hint status --deserialize=../hint.dat -uno >hint.d1_no && + test_cmp expect.use_u hint.d1_no && + git -C hint -c status.showuntrackedfiles=no status --deserialize=../hint.dat >hint.d2_no && + test_cmp expect.use_u hint.d2_no + +' + +test_expect_success 'ensure deserialize -v does not crash' ' + + git init -b main verbose_test && + touch verbose_test/a && + touch verbose_test/b && + touch verbose_test/c && + git -C verbose_test add a b c && + git -C verbose_test commit -m abc && + + echo green >>verbose_test/a && + git -C verbose_test add a && + echo red_1 >>verbose_test/b && + echo red_2 >verbose_test/dirt && + + git -C verbose_test status >output.ref && + git -C verbose_test status -v >output.ref_v && + + git -C verbose_test --no-optional-locks status --serialize=../verbose_test.dat >output.ser.long && + git -C verbose_test --no-optional-locks status --serialize=../verbose_test.dat_v -v >output.ser.long_v && + + # Verify that serialization does not affect the status output itself. + test_cmp output.ref output.ser.long && + test_cmp output.ref_v output.ser.long_v && + + GIT_TRACE2_PERF="$(pwd)"/verbose_test.log \ + git -C verbose_test status --deserialize=../verbose_test.dat >output.des.long && + + # Verify that normal deserialize was actually used and produces the same result. + test_cmp output.ser.long output.des.long && + test_grep -q "deserialize/result:ok" verbose_test.log && + + GIT_TRACE2_PERF="$(pwd)"/verbose_test.log_v \ + git -C verbose_test status --deserialize=../verbose_test.dat_v -v >output.des.long_v && + + # Verify that vebose mode produces the same result because verbose was rejected. + test_cmp output.ser.long_v output.des.long_v && + test_grep -q "deserialize/reject:args/verbose" verbose_test.log_v +' + +test_expect_success 'fallback when implicit' ' + git init -b main implicit_fallback_test && + git -C implicit_fallback_test -c status.deserializepath=foobar status +' + +test_expect_success 'fallback when explicit' ' + git init -b main explicit_fallback_test && + git -C explicit_fallback_test status --deserialize=foobar +' + +test_expect_success 'deserialize from stdin' ' + git init -b main stdin_test && + git -C stdin_test status --serialize >serialized_status.dat && + cat serialize_status.dat | git -C stdin_test status --deserialize +' + +test_done diff --git a/t/t7523-status-complete-untracked.sh b/t/t7523-status-complete-untracked.sh new file mode 100755 index 00000000000000..c5c03d76350f9f --- /dev/null +++ b/t/t7523-status-complete-untracked.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +test_description='git status untracked complete tests' + +. ./test-lib.sh + +test_expect_success 'setup' ' + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt +' + +test_expect_success 'verify untracked-files=complete' ' + cat >expect <<-\EOF && + ? expect + ? output + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --porcelain=v2 --untracked-files=complete --ignored >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_done diff --git a/t/t7525-status-rename.sh b/t/t7525-status-rename.sh index d409de1a33fd79..0e2634a5f7e5bb 100755 --- a/t/t7525-status-rename.sh +++ b/t/t7525-status-rename.sh @@ -97,6 +97,38 @@ test_expect_success 'status score=01%' ' test_grep "renamed:" actual ' +test_expect_success 'diff.renameThreshold sets default threshold' ' + git -c diff.renameThreshold=100% status >actual && + test_grep "deleted:" actual && + test_grep "new file:" actual +' + +test_expect_success 'status.renameThreshold overrides diff.renameThreshold' ' + git -c diff.renameThreshold=100% -c status.renameThreshold=01% status >actual && + test_grep "renamed:" actual +' + +test_expect_success 'diff.renameThreshold=01% detects rename in status' ' + git -c diff.renameThreshold=01% status >actual && + test_grep "renamed:" actual +' + +test_expect_success 'commit honors diff.renameThreshold' ' + git -c diff.renameThreshold=100% commit --dry-run >actual && + test_grep "deleted:" actual && + test_grep "new file:" actual +' + +test_expect_success 'commit honors status.renameThreshold' ' + git -c status.renameThreshold=01% commit --dry-run >actual && + test_grep "renamed:" actual +' + +test_expect_success '-M overrides status.renameThreshold' ' + git -c status.renameThreshold=100% status -M=01% >actual && + test_grep "renamed:" actual +' + test_expect_success 'copies not overridden by find-renames' ' cp renamed copy && git add copy && diff --git a/t/t7616-merge-sparse-checkout.sh b/t/t7616-merge-sparse-checkout.sh new file mode 100755 index 00000000000000..5ce12431f62ad1 --- /dev/null +++ b/t/t7616-merge-sparse-checkout.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +test_description='merge can handle sparse-checkout' + +. ./test-lib.sh + +# merges with conflicts + +test_expect_success 'setup' ' + git branch -M main && + test_commit a && + test_commit file && + git checkout -b delete-file && + git rm file.t && + test_tick && + git commit -m "remove file" && + git checkout main && + test_commit modify file.t changed +' + +test_expect_success 'merge conflict deleted file and modified' ' + echo "/a.t" >.git/info/sparse-checkout && + test_config core.sparsecheckout true && + git checkout -f && + test_path_is_missing file.t && + test_must_fail git merge delete-file && + test_path_is_file file.t && + test "changed" = "$(cat file.t)" +' + +test_done diff --git a/t/t7817-grep-sparse-checkout.sh b/t/t7817-grep-sparse-checkout.sh index eb595645657fad..db3004c4fe71c0 100755 --- a/t/t7817-grep-sparse-checkout.sh +++ b/t/t7817-grep-sparse-checkout.sh @@ -49,7 +49,7 @@ test_expect_success 'setup' ' echo "text" >B/b && git add A B && git commit -m sub && - git sparse-checkout init --cone && + git sparse-checkout init --cone --no-sparse-index && git sparse-checkout set B ) && diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 4f65fa9439c0b8..6117640a73b3b3 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -37,6 +37,25 @@ test_systemd_analyze_verify () { fi } +test_import_packfile () { + printf "blob\ndata <actual && test_grep "usage: git maintenance " actual && @@ -444,6 +463,40 @@ test_expect_success 'maintenance.loose-objects.batchSize' ' test_must_be_empty err ' +test_expect_success 'loose-objects and gvfs.sharedCache' ' + git init gvfs-worktree && + git init --bare gvfs-shared && + ( + cd gvfs-worktree && + git config gvfs.sharedCache "$PWD/../gvfs-shared/objects" && + + # Hack to stop maintenance from running during "git commit" + echo in use >.git/objects/maintenance.lock && + git config maintenance.loose-objects.auto 1 && + test_commit create-loose-object && + rm .git/objects/maintenance.lock && + ! ls -l ../gvfs-shared/objects/??/* && + ls -l .git/objects/??/* >loose-objects && + test_file_not_empty loose-objects && + ! ls -l ../gvfs-shared/objects/pack/*.pack && + + # move the loose objects into the shared objects as if they had been + # fetched via the `gvfs-helper` + mv .git/objects/?? ../gvfs-shared/objects/ && + + # Run `loose-objects` twice: The first run creates a pack-file + # but does not delete loose objects, the second run deletes + # loose objects but does not create a pack-file. + git maintenance run --task=loose-objects && + git maintenance run --task=loose-objects && + + ! ls -l .git/objects/??/* && + ! ls -l ../gvfs-shared/.git/objects/??/* && + ls -l ../gvfs-shared/objects/pack/*.pack >shared-packs && + test_file_not_empty shared-packs + ) +' + test_expect_success 'incremental-repack task' ' packDir=.git/objects/pack && for i in $(test_seq 1 5) @@ -1126,6 +1179,7 @@ test_expect_success '--schedule inheritance weekly -> daily -> hourly' ' git maintenance run --schedule=weekly 2>/dev/null && test_maintenance_tasks weekly.txt <<-\EOF pack-refs foreground + cache-local-objects foreground prefetch loose-objects incremental-repack @@ -1163,6 +1217,7 @@ test_expect_success 'maintenance.strategy inheritance' ' test_maintenance_tasks incremental-weekly.txt <<-\EOF && pack-refs foreground + cache-local-objects foreground prefetch loose-objects incremental-repack @@ -1218,6 +1273,7 @@ test_expect_success 'maintenance.strategy is respected' ' test_strategy incremental --schedule=weekly <<-\EOF && pack-refs foreground + cache-local-objects foreground prefetch loose-objects incremental-repack @@ -1791,4 +1847,118 @@ test_expect_success 'maintenance aborts with existing lock file' ' test_grep "Another scheduled git-maintenance(1) process seems to be running" err ' +test_expect_success 'cache-local-objects task with no shared cache no op' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + + test_commit something && + git config set maintenance.gc.enabled false && + git config set maintenance.geometric-repack.enabled false && + git config set maintenance.cache-local-objects.enabled true && + git config set maintenance.cache-local-objects.auto 1 && + + test_import_packfile && + test_get_packdir_files "*.pack" "*.idx" "*.keep" "*.rev" \ + >files.txt && + test_get_loose_object_files >>files.txt && + + git maintenance run && + while IFS= read -r f; do + test_path_exists $f || exit 1 + done files.txt && + test_get_loose_object_files >>files.txt && + + git maintenance run && + while IFS= read -r f; do + test_path_exists $f || exit 1 + done src.txt && + test_get_loose_object_files >>src.txt && + + rm -f .git/objects/pack/*.rev .git/objects/pack/*.keep && + + sed "s/.git\\/objects\\//..\\/cache\\//" src.txt >dst.txt && + + git maintenance run && + while IFS= read -r f; do + test_path_is_missing $f || exit 1 + done src.txt && + test_get_loose_object_files >>src.txt && + + sed "s/.git\\/objects\\//..\\/cache\\//" src.txt >dst.txt && + + git maintenance run && + while IFS= read -r f; do + test_path_is_missing $f || exit 1 + done v1-before-inexact.txt && + test_write_lines other1 other2 other3 >unrelated.txt && + git add v1-before-inexact.txt unrelated.txt && + GIT_AUTHOR_NAME=Original git commit -m "add files" && + + test_write_lines changed1 line2 line3 >v2-before-exact.txt && + git rm v1-before-inexact.txt && + git rm unrelated.txt && + git add v2-before-exact.txt && + GIT_AUTHOR_NAME=Inexact git commit -m "inexact rename with content change" && + + git mv v2-before-exact.txt v3.txt && + GIT_AUTHOR_NAME=Exact git commit -m "exact rename" +' + +test_expect_success 'blame follows renames by default' ' + git blame --porcelain v3.txt >output && + test_grep "^filename v1-before-inexact.txt" output +' + +test_expect_success 'blame.renames=false disables rename following' ' + git -c blame.renames=false blame --porcelain v3.txt >output && + test_grep ! "^filename v1-before-inexact.txt" output && + test_grep ! "^filename v2-before-exact.txt" output +' + +test_expect_success 'blame.renameThreshold=100% allows exact but skips inexact renames' ' + git -c blame.renameThreshold=100% blame --porcelain v3.txt >output && + test_grep "^filename v2-before-exact.txt" output && + test_grep ! "^filename v1-before-inexact.txt" output +' + +test_expect_success 'blame.renameLimit=1 skips when sources*destinations exceeds limit' ' + git -c blame.renameLimit=1 blame --porcelain v3.txt >output && + test_grep "^filename v2-before-exact.txt" output && + test_grep ! "^filename v1-before-inexact.txt" output +' + +test_expect_success 'blame.renameLimit=2 detects with two sources' ' + git -c blame.renameLimit=2 blame --porcelain v3.txt >output && + test_grep "^filename v1-before-inexact.txt" output +' + +test_done diff --git a/t/t9210-scalar.sh b/t/t9210-scalar.sh index c8463d0ac4f401..0d401a4f676fce 100755 --- a/t/t9210-scalar.sh +++ b/t/t9210-scalar.sh @@ -7,6 +7,13 @@ test_description='test the `scalar` command' GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt,launchctl:true,schtasks:true" export GIT_TEST_MAINT_SCHEDULER +# Do not write any files outside the trash directory +Scalar_UNATTENDED=1 +export Scalar_UNATTENDED + +GIT_ASKPASS=true +export GIT_ASKPASS + test_expect_success 'scalar shows a usage' ' test_expect_code 129 scalar -h ' @@ -274,6 +281,29 @@ test_expect_success 'scalar reconfigure --all with detached HEADs' ' done ' +test_expect_success 'verify http..version=HTTP/1.1 for ADO URLs' ' + test_when_finished rm -rf test-http-url-config && + + # Create a test repository + git init test-http-url-config && + + # Test both URL types + for url in "https://test@dev.azure.com/test/project/_git/repo" \ + "https://contoso.visualstudio.com/project/_git/repo" + do + # Set URL as remote + git -C test-http-url-config config set remote.origin.url "$url" && + + # Run scalar reconfigure + scalar reconfigure test-http-url-config && + + # Verify URL-specific HTTP version setting + git -C test-http-url-config config "http.$url.version" >actual && + echo "HTTP/1.1" >expect && + test_cmp expect actual || return 1 + done +' + test_expect_success '`reconfigure -a` removes stale config entries' ' git init stale/src && scalar register stale && @@ -333,4 +363,339 @@ test_expect_success UNZIP 'scalar diagnose' ' test_grep "^Total: [1-9]" out ' +GIT_TEST_ALLOW_GVFS_VIA_HTTP=1 +export GIT_TEST_ALLOW_GVFS_VIA_HTTP + +test_set_port GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT +ORIGIN_HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT +ORIGIN_PID_FILE="$(pwd)"/pid-file.$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT.pid +ORIGIN_SERVER_LOG="$(pwd)"/server-origin.$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT.log + +test_atexit ' + test -f "$ORIGIN_PID_FILE" || return 0 + + # The server will shutdown automatically when we delete the pid-file. + rm -f "$ORIGIN_PID_FILE" + + test -z "$verbose$verbose_log" || { + echo "server log:" + cat "$ORIGIN_SERVER_LOG" + } + + # Give it a few seconds to shutdown (mainly to completely release the + # port before the next test start another instance and it attempts to + # bind to it). + for k in $(test_seq 5) + do + grep -q "Starting graceful shutdown" "$ORIGIN_SERVER_LOG" && + return 0 || + sleep 1 + done + + echo "stop_gvfs_protocol_server: timeout waiting for server shutdown" + return 1 +' + +start_gvfs_enabled_http_server () { + GIT_HTTP_EXPORT_ALL=1 \ + test-gvfs-protocol --verbose \ + --listen=127.0.0.1 \ + --port=$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT \ + --reuseaddr \ + --pid-file="$ORIGIN_PID_FILE" \ + 2>"$ORIGIN_SERVER_LOG" & + + for k in $(test_seq 5) + do + if test -f "$ORIGIN_PID_FILE" + then + return 0 + fi + sleep 1 + done + return 1 +} + +test_expect_success 'start GVFS-enabled server' ' + git config uploadPack.allowFilter false && + git config uploadPack.allowAnySHA1InWant false && + start_gvfs_enabled_http_server +' + +test_expect_success '`scalar clone` with GVFS-enabled server' ' + : the fake cache server requires fake authentication && + git config --global core.askPass true && + + # We must set credential.interactive=true to bypass a setting + # in "scalar clone" that disables interactive credentials during + # an unattended command. + GIT_TRACE2_EVENT="$(pwd)/clone-trace-with-gvfs" scalar \ + -c credential.interactive=true \ + clone --gvfs-protocol \ + --single-branch -- http://$ORIGIN_HOST_PORT/ using-gvfs && + + grep "GET/config(main)" expect && + git -C using-gvfs/src config gvfs.sharedCache >actual && + test_cmp expect actual && + + : verify that URL-specific HTTP version setting is configured for GVFS URLs in clone && + git -C using-gvfs/src config "http.http://$ORIGIN_HOST_PORT/.version" >actual && + echo "HTTP/1.1" >expect && + test_cmp expect actual && + + second=$(git rev-parse --verify second:second.t) && + ( + cd using-gvfs/src && + test_path_is_missing 1/2 && + GIT_TRACE=$PWD/trace.txt git cat-file blob $second >actual && + : verify that the gvfs-helper was invoked to fetch it && + test_grep gvfs-helper trace.txt && + echo "second" >expect && + test_cmp expect actual + ) +' + +test_expect_success '`scalar clone --no-prefetch` skips the initial prefetch' ' + git config --global core.askPass true && + tip=$(git rev-parse HEAD) && + + # A normal GVFS-enabled clone issues a "/gvfs/prefetch" request, + # which shows up in the trace as a "prefetch/since" data event. + GIT_TRACE2_EVENT="$(pwd)/with-prefetch-trace" scalar \ + -c credential.interactive=true \ + clone --gvfs-protocol --single-branch \ + -- http://$ORIGIN_HOST_PORT/ with-prefetch && + test_grep "prefetch/since" with-prefetch-trace && + + # ... but "--no-prefetch" skips that request during the clone while + # fetching the tip commit and its trees through the objects POST + # endpoint before checkout. + GIT_TRACE2_EVENT="$(pwd)/no-prefetch-trace" \ + GIT_TRACE2_PERF="$(pwd)/no-prefetch-perf" scalar \ + -c credential.interactive=true \ + clone --no-prefetch --gvfs-protocol --single-branch \ + -- http://$ORIGIN_HOST_PORT/ no-prefetch && + test_grep ! "prefetch/since" no-prefetch-trace && + test_grep "gh_client__queue_oid: $tip" no-prefetch-perf && + test_trace2_data gh-client objects/post/nr_objects 1 \ + expect && + git -C no-prefetch/src config core.gvfs >actual && + test_cmp expect actual && + + : and a subsequent git fetch performs the deferred prefetch && + GIT_TRACE2_EVENT="$(pwd)/fetch-trace" \ + git -C no-prefetch/src fetch origin && + test_grep "prefetch/since" fetch-trace +' + +test_expect_success '`scalar clone` with GVFS-enabled server; local cache path' ' + : the fake cache server requires fake authentication && + git config --global core.askPass true && + + LOCAL_CACHE_BASE="$(pwd)/local" && + + # We must set credential.interactive=true to bypass a setting + # in "scalar clone" that disables interactive credentials during + # an unattended command. + scalar \ + -c credential.interactive=true \ + clone --gvfs-protocol \ + --local-cache-path="$LOCAL_CACHE_BASE" \ + --single-branch -- http://$ORIGIN_HOST_PORT/ with-local && + + : verify that the shared cache has been configured && + cache_key="url_$(printf "%s" http://$ORIGIN_HOST_PORT/ | + tr A-Z a-z | + test-tool sha1)" && + LOCAL_CACHE_DIR="$LOCAL_CACHE_BASE/$cache_key" && + echo "$LOCAL_CACHE_DIR" >expect && + git -C with-local/src config gvfs.sharedCache >actual && + test_cmp expect actual && + + : check the local cache is recreated on fetch && + rm -rf $LOCAL_CACHE_BASE && + git -C with-local fetch && + test_path_is_dir "$LOCAL_CACHE_DIR" && + test_path_is_dir "$LOCAL_CACHE_DIR/pack" +' + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +test_expect_success 'scalar clone: all verbs with different servers' ' + git config --global core.askPass true && + + test_when_finished "per_test_cleanup" && + test_when_finished "scalar delete scalar-clone" && + + start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && + start_gvfs_protocol_server 3 && + start_gvfs_protocol_server 4 && + + # Configure each verb to use a different server: + # - server 1: default (unused in this test; not running.) + # - server 2: prefetch + # - server 3: get + # - server 4: post + scalar -c credential.interactive=true \ + clone --full-clone \ + --cache-server-url="$(cache_server_url 1)" \ + --prefetch-cache-server-url="$(cache_server_url 2)" \ + --get-cache-server-url="$(cache_server_url 3)" \ + --post-cache-server-url="$(cache_server_url 4)" \ + --gvfs-protocol \ + -- "http://$ORIGIN_HOST_PORT/" scalar-clone 2>err >out && + + test_grep "Cache server URL: $(cache_server_url 1)" err && + test_grep "Prefetch cache server URL: $(cache_server_url 2)" err && + test_grep "Objects GET cache server URL: $(cache_server_url 3)" err && + test_grep "Objects POST cache server URL: $(cache_server_url 4)" err && + + test_cmp_config -C scalar-clone/src "$(cache_server_url 1)" gvfs.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 2)" gvfs.prefetch.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 3)" gvfs.get.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 4)" gvfs.post.cache-server && + + verify_server_was_contacted 1 && + verify_server_was_contacted 2 && + verify_server_was_contacted 3 +' + +test_expect_success EXPENSIVE 'fetch does not hang in gvfs-helper' ' + # Marked as EXPENSIVE as this will go through multiple rounds of + # exponential backoff, including delays of 8, 16, 32, 64, 128, + # and 256 seconds in two separate instances. + test_must_fail git -C using-gvfs/src fetch origin does-not-exist +' + +test_expect_success '`scalar clone --no-gvfs-protocol` skips gvfs/config' ' + # the fake cache server requires fake authentication && + git config --global core.askPass true && + + # We must set credential.interactive=true to bypass a setting + # in "scalar clone" that disables interactive credentials during + # an unattended command. + GIT_TRACE2_EVENT="$(pwd)/clone-trace-no-gvfs" scalar \ + -c credential.interactive=true \ + clone --no-gvfs-protocol \ + --single-branch -- http://$ORIGIN_HOST_PORT/ skipping-gvfs && + + ! grep "GET/config(main)" scalar.repos && + test_grep ! -F "$(pwd)/test-repo/src" scalar.repos && + + : at enlistment root, i.e. parent of repository, is supported && + GIT_CEILING_DIRECTORIES="$(pwd)" scalar register test-repo && + git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/test-repo/src" && + scalar list >scalar.repos && + test_grep -F "$(pwd)/test-repo/src" scalar.repos && + + : scalar delete properly unregisters enlistment && + scalar delete test-repo && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/test-repo/src" && + scalar list >scalar.repos && + test_grep ! -F "$(pwd)/test-repo/src" scalar.repos +' + +test_expect_success '`scalar register` & `unregister` with existing repo' ' + git init existing && + scalar register existing && + git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + test_grep -F "$(pwd)/existing" scalar.repos && + scalar unregister existing && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + test_grep ! -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar unregister` with existing repo, deleted .git' ' + scalar register existing && + rm -rf existing/.git && + scalar unregister existing && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + test_grep ! -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar register` existing repo with `src` folder' ' + git init existing && + mkdir -p existing/src && + scalar register existing/src && + scalar list >scalar.repos && + test_grep -F "$(pwd)/existing" scalar.repos && + scalar unregister existing && + scalar list >scalar.repos && + test_grep ! -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar delete` with existing repo' ' + git init existing && + scalar register existing && + scalar delete existing && + test_path_is_missing existing +' + +test_expect_success 'scalar cache-server basics' ' + repo=with-cache-server && + git init $repo && + scalar cache-server --get $repo >out && + cat >expect <<-EOF && + Using cache server: (undefined) + EOF + test_cmp expect out && + + scalar cache-server --set http://fake-server/url $repo && + test_cmp_config -C $repo http://fake-server/url gvfs.cache-server && + scalar delete $repo && + test_path_is_missing $repo +' + +test_expect_success 'scalar cache-server list URL' ' + repo=with-real-gvfs && + git init $repo && + git -C $repo remote add origin http://$ORIGIN_HOST_PORT/ && + scalar cache-server --list origin $repo >out && + + cat >expect <<-EOF && + #0: http://$ORIGIN_HOST_PORT/servertype/cache + EOF + + test_cmp expect out && + + test_must_fail scalar -C $repo cache-server --list 2>err && + test_grep "requires a value" err && + + scalar delete $repo && + test_path_is_missing $repo +' + test_done diff --git a/t/t9211-scalar-clone.sh b/t/t9211-scalar-clone.sh index 002d6ecdc12b47..90106928660c1a 100755 --- a/t/t9211-scalar-clone.sh +++ b/t/t9211-scalar-clone.sh @@ -206,4 +206,34 @@ test_expect_success '`scalar clone --no-src`' ' test_cmp with without ' +test_expect_success '`scalar clone --ref-format`' ' + scalar clone "file://$(pwd)/to-clone" refs-default && + scalar clone --ref-format files "file://$(pwd)/to-clone" refs-files && + scalar clone --ref-format reftable "file://$(pwd)/to-clone" refs-reftable && + + test_path_is_dir refs-default/src && + test_path_is_dir refs-files/src && + test_path_is_dir refs-reftable/src && + + ( + cd refs-default/src && + case test_detect_ref_format in + files) + test_must_fail git config --local extensions.refstorage + ;; + reftable) + test_cmp_config reftable extensions.refstorage + ;; + esac + ) && + ( + cd refs-files/src && + test_must_fail git config --local extensions.refstorage + ) && + ( + cd refs-reftable/src && + test_cmp_config reftable extensions.refstorage + ) +' + test_done diff --git a/t/unit-tests/u-ctype.c b/t/unit-tests/u-ctype.c index 32e65867cdc28d..51a6c27ca45ef6 100644 --- a/t/unit-tests/u-ctype.c +++ b/t/unit-tests/u-ctype.c @@ -33,70 +33,70 @@ void test_ctype__isspace(void) { - TEST_CHAR_CLASS(isspace, " \n\r\t"); + TEST_CHAR_CLASS(isspace, " \n\r\t"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isdigit(void) { - TEST_CHAR_CLASS(isdigit, DIGIT); + TEST_CHAR_CLASS(isdigit, DIGIT); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isalpha(void) { - TEST_CHAR_CLASS(isalpha, LOWER UPPER); + TEST_CHAR_CLASS(isalpha, LOWER UPPER); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isalnum(void) { - TEST_CHAR_CLASS(isalnum, LOWER UPPER DIGIT); + TEST_CHAR_CLASS(isalnum, LOWER UPPER DIGIT); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__is_glob_special(void) { - TEST_CHAR_CLASS(is_glob_special, "*?[\\"); + TEST_CHAR_CLASS(is_glob_special, "*?[\\"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__is_regex_special(void) { - TEST_CHAR_CLASS(is_regex_special, "$()*+.?[\\^{|"); + TEST_CHAR_CLASS(is_regex_special, "$()*+.?[\\^{|"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__is_pathspec_magic(void) { - TEST_CHAR_CLASS(is_pathspec_magic, "!\"#%&',-/:;<=>@_`~"); + TEST_CHAR_CLASS(is_pathspec_magic, "!\"#%&',-/:;<=>@_`~"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isascii(void) { - TEST_CHAR_CLASS(isascii, ASCII); + TEST_CHAR_CLASS(isascii, ASCII); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__islower(void) { - TEST_CHAR_CLASS(islower, LOWER); + TEST_CHAR_CLASS(islower, LOWER); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isupper(void) { - TEST_CHAR_CLASS(isupper, UPPER); + TEST_CHAR_CLASS(isupper, UPPER); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__iscntrl(void) { - TEST_CHAR_CLASS(iscntrl, CNTRL); + TEST_CHAR_CLASS(iscntrl, CNTRL); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__ispunct(void) { - TEST_CHAR_CLASS(ispunct, PUNCT); + TEST_CHAR_CLASS(ispunct, PUNCT); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isxdigit(void) { - TEST_CHAR_CLASS(isxdigit, DIGIT "abcdefABCDEF"); + TEST_CHAR_CLASS(isxdigit, DIGIT "abcdefABCDEF"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isprint(void) { - TEST_CHAR_CLASS(isprint, LOWER UPPER DIGIT PUNCT " "); + TEST_CHAR_CLASS(isprint, LOWER UPPER DIGIT PUNCT " "); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index a8da278c681cca..3953df300d10e5 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -257,11 +257,11 @@ void test_odb_inmemory__freshen_object(void) const char *end; cl_must_pass(parse_oid_hex_algop(RANDOM_OID, &oid, &end, repo.hash_algo)); - cl_assert_equal_i(odb_source_freshen_object(&source->base, &oid, NULL), 0); + cl_assert_equal_i(odb_source_freshen_object(&source->base, &oid, NULL, 0), 0); cl_assert_write_object(source, "foobar", OBJ_BLOB, &written_oid); cl_assert_equal_i(odb_source_freshen_object(&source->base, - &written_oid, NULL), 1); + &written_oid, NULL, 0), 1); odb_source_free(&source->base); } diff --git a/trace2.c b/trace2.c index c23c0a227b7032..24dd7b1be9fbbe 100644 --- a/trace2.c +++ b/trace2.c @@ -227,6 +227,16 @@ void trace2_initialize_fl(const char *file, int line) if (!tr2_tgt_want_builtins()) return; trace2_enabled = 1; + + /* + * getenv() on Windows stomps on `errno` and the code in + * tr2_dst.c verifies that warnings are enabled before + * formatting the warning message (and calling strerror()). + * So prefetch the value from the environment before we need + * it. + */ + tr2_dst_want_warning(); + if (!git_env_bool("GIT_TRACE2_REDACT", 1)) trace2_redact = 0; diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c index 5be892cd5cdefa..61579f24bdbde3 100644 --- a/trace2/tr2_dst.c +++ b/trace2/tr2_dst.c @@ -24,7 +24,7 @@ */ static int tr2env_max_files = 0; -static int tr2_dst_want_warning(void) +int tr2_dst_want_warning(void) { static int tr2env_dst_debug = -1; diff --git a/trace2/tr2_dst.h b/trace2/tr2_dst.h index b1a8c144e073ba..4166539eb9e100 100644 --- a/trace2/tr2_dst.h +++ b/trace2/tr2_dst.h @@ -35,4 +35,16 @@ int tr2_dst_trace_want(struct tr2_dst *dst); */ void tr2_dst_write_line(struct tr2_dst *dst, struct strbuf *buf_line); +/* + * Return true if we want warning messages when trying to open a + * destination. + * + * (Trace2 always silently fails if a target cannot be opened so that + * we don't affect the execution of the Git command, but it is helpful + * for debugging telemetry configuration if we log warning messages + * when trying to open a target. This is controlled by another config + * value.) + */ +int tr2_dst_want_warning(void); + #endif /* TR2_DST_H */ diff --git a/trace2/tr2_tbuf.c b/trace2/tr2_tbuf.c index c3b3822ed7e4af..ef57376f3c3e24 100644 --- a/trace2/tr2_tbuf.c +++ b/trace2/tr2_tbuf.c @@ -3,45 +3,64 @@ void tr2_tbuf_local_time(struct tr2_tbuf *tb) { - struct timeval tv; - struct tm tm; + struct timeval tv = { 0 }; + struct tm tm = { 0 }; time_t secs; + int len; gettimeofday(&tv, NULL); secs = tv.tv_sec; localtime_r(&secs, &tm); - xsnprintf(tb->buf, sizeof(tb->buf), "%02d:%02d:%02d.%06ld", tm.tm_hour, - tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + len = snprintf(tb->buf, sizeof(tb->buf), "%02d:%02d:%02d.%06ld", + tm.tm_hour, tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + + if (len < 0 || (size_t)len >= sizeof(tb->buf)) { + const char *blank = "00:00:00.000000"; + strlcpy(tb->buf, blank, sizeof(tb->buf)); + } } void tr2_tbuf_utc_datetime_extended(struct tr2_tbuf *tb) { - struct timeval tv; - struct tm tm; + struct timeval tv = { 0 }; + struct tm tm = { 0 }; time_t secs; + int len; gettimeofday(&tv, NULL); secs = tv.tv_sec; gmtime_r(&secs, &tm); - xsnprintf(tb->buf, sizeof(tb->buf), - "%4d-%02d-%02dT%02d:%02d:%02d.%06ldZ", tm.tm_year + 1900, - tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, - (long)tv.tv_usec); + len = snprintf(tb->buf, sizeof(tb->buf), + "%4d-%02d-%02dT%02d:%02d:%02d.%06ldZ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + + if (len < 0 || (size_t)len >= sizeof(tb->buf)) { + const char *blank = "1900-00-00T00:00:00.000000Z"; + strlcpy(tb->buf, blank, sizeof(tb->buf)); + } } void tr2_tbuf_utc_datetime(struct tr2_tbuf *tb) { - struct timeval tv; - struct tm tm; + struct timeval tv = { 0 }; + struct tm tm = { 0 }; time_t secs; + int len; gettimeofday(&tv, NULL); secs = tv.tv_sec; gmtime_r(&secs, &tm); - xsnprintf(tb->buf, sizeof(tb->buf), "%4d%02d%02dT%02d%02d%02d.%06ldZ", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, - tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + len = snprintf(tb->buf, sizeof(tb->buf), + "%4d%02d%02dT%02d%02d%02d.%06ldZ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + + if (len < 0 || (size_t)len >= sizeof(tb->buf)) { + const char *blank = "19000000T000000.000000Z"; + strlcpy(tb->buf, blank, sizeof(tb->buf)); + } } diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index 5a0381791f7eb4..5ecc0b920e3694 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -39,7 +39,7 @@ static struct tr2_dst tr2dst_event = { * event target. Use the TR2_SYSENV_EVENT_NESTING setting to increase * region details in the event target. */ -static int tr2env_event_max_nesting_levels = 2; +static int tr2env_event_max_nesting_levels = 4; /* * Use the TR2_SYSENV_EVENT_BRIEF to omit the