#
cargo/sources/git/
oxide.rs

    1//! This module contains all code sporting `gitoxide` for operations on `git` repositories and it mirrors
    2//! `utils` closely for now. One day it can be renamed into `utils` once `git2` isn't required anymore.
    3
    4use crate::util::HumanBytes;
    5use crate::util::network::http::HttpTimeout;
    6use crate::util::{MetricsCounter, Progress, network};
    7use crate::{CargoResult, GlobalContext};
    8use cargo_util::paths;
    9use gix::bstr::{BString, ByteSlice};
    10use std::cell::RefCell;
    11use std::path::Path;
    12use std::sync::atomic::{AtomicBool, Ordering};
    13use std::sync::{Arc, Weak};
    14use std::time::{Duration, Instant};
    15use tracing::debug;
    16
    17/// For the time being, `repo_path` makes it easy to instantiate a gitoxide repo just for fetching.
    18/// In future this may change to be the gitoxide repository itself.
    19pub fn with_retry_and_progress(
    20    repo_path: &std::path::Path,
    21    gctx: &GlobalContext,
    22    repo_remote_url: &str,
    23    cb: &(
    24         dyn Fn(
    25        &std::path::Path,
    26        &AtomicBool,
    27        &mut gix::progress::tree::Item,
    28        &mut dyn FnMut(&gix::bstr::BStr),
    29    ) -> Result<(), crate::sources::git::fetch::Error>
    30             + Send
    31             + Sync
    32     ),
    33) -> CargoResult<()> {
    34    std::thread::scope(|s| {
    35        let mut progress_bar = Progress::new("Fetch", gctx);
    36        let is_shallow = gctx.cli_unstable().git.map_or(false, |features| {
    37            features.shallow_deps || features.shallow_index
    38        });
    39        network::retry::with_retry(gctx, || {
    40            let progress_root: Arc<gix::progress::tree::Root> =
    41                gix::progress::tree::root::Options {
    42                    initial_capacity: 10,
    43                    message_buffer_capacity: 10,
    44                }
    45                .into();
    46            let root = Arc::downgrade(&progress_root);
    47            let thread = s.spawn(move || {
    48                let mut progress = progress_root.add_child("operation");
    49                let mut urls = RefCell::new(Default::default());
    50                let res = cb(
    51                    &repo_path,
    52                    &AtomicBool::default(),
    53                    &mut progress,
    54                    &mut |url| {
    55                        *urls.borrow_mut() = Some(url.to_owned());
    56                    },
    57                );
    58                amend_authentication_hints(res, repo_remote_url, urls.get_mut().take())
    59            });
    60            translate_progress_to_bar(&mut progress_bar, root, is_shallow)?;
    61            thread.join().expect("no panic in scoped thread")
    62        })
    63    })
    64}
    65
    66fn translate_progress_to_bar(
    67    progress_bar: &mut Progress<'_>,
    68    root: Weak<gix::progress::tree::Root>,
    69    is_shallow: bool,
    70) -> CargoResult<()> {
    71    let remote_progress: gix::progress::Id = gix::remote::fetch::ProgressId::RemoteProgress.into();
    72    let read_pack_bytes: gix::progress::Id =
    73        gix::odb::pack::bundle::write::ProgressId::ReadPackBytes.into();
    74    let delta_index_objects: gix::progress::Id =
    75        gix::odb::pack::index::write::ProgressId::IndexObjects.into();
    76    let resolve_objects: gix::progress::Id =
    77        gix::odb::pack::index::write::ProgressId::ResolveObjects.into();
    78
    79    // We choose `N=10` here to make a `300ms * 10slots ~= 3000ms`
    80    // sliding window for tracking the data transfer rate (in bytes/s).
    81    let mut last_percentage_update = Instant::now();
    82    let mut last_fast_update = Instant::now();
    83    let mut counter = MetricsCounter::<10>::new(0, last_percentage_update);
    84
    85    let mut tasks = Vec::with_capacity(10);
    86    let slow_check_interval = std::time::Duration::from_millis(300);
    87    let fast_check_interval = Duration::from_millis(50);
    88    let sleep_interval = Duration::from_millis(10);
    89    debug_assert_eq!(
    90        slow_check_interval.as_millis() % fast_check_interval.as_millis(),
    91        0,
    92        "progress should be smoother by keeping these as multiples of each other"
    93    );
    94    debug_assert_eq!(
    95        fast_check_interval.as_millis() % sleep_interval.as_millis(),
    96        0,
    97        "progress should be smoother by keeping these as multiples of each other"
    98    );
    99
    100    let num_phases = if is_shallow { 3 } else { 2 }; // indexing + delta-resolution, both with same amount of objects to handle
    101    while let Some(root) = root.upgrade() {
    102        std::thread::sleep(sleep_interval);
    103        let needs_update = last_fast_update.elapsed() >= fast_check_interval;
    104        if !needs_update {
    105            continue;
    106        }
    107        let now = Instant::now();
    108        last_fast_update = now;
    109
    110        root.sorted_snapshot(&mut tasks);
    111
    112        fn progress_by_id(
    113            id: gix::progress::Id,
    114            task: &gix::progress::Task,
    115        ) -> Option<(&str, &gix::progress::Value)> {
    116            (task.id == id)
    117                .then(|| task.progress.as_ref())
    118                .flatten()
    119                .map(|value| (task.name.as_str(), value))
    120        }
    121        fn find_in<K>(
    122            tasks: &[(K, gix::progress::Task)],
    123            cb: impl Fn(&gix::progress::Task) -> Option<(&str, &gix::progress::Value)>,
    124        ) -> Option<(&str, &gix::progress::Value)> {
    125            tasks.iter().find_map(|(_, t)| cb(t))
    126        }
    127
    128        if let Some((_, objs)) = find_in(&tasks, |t| progress_by_id(resolve_objects, t)) {
    129            // Phase 3: Resolving deltas.
    130            let objects = objs.step.load(Ordering::Relaxed);
    131            let total_objects = objs.done_at.expect("known amount of objects");
    132            let msg = format!(", ({objects}/{total_objects}) resolving deltas");
    133
    134            progress_bar.tick(
    135                (total_objects * (num_phases - 1)) + objects,
    136                total_objects * num_phases,
    137                &msg,
    138            )?;
    139        } else if let Some((objs, read_pack)) =
    140            find_in(&tasks, |t| progress_by_id(read_pack_bytes, t)).and_then(|read| {
    141                find_in(&tasks, |t| progress_by_id(delta_index_objects, t))
    142                    .map(|delta| (delta.1, read.1))
    143            })
    144        {
    145            // Phase 2: Receiving objects.
    146            let objects = objs.step.load(Ordering::Relaxed);
    147            let total_objects = objs.done_at.expect("known amount of objects");
    148            let received_bytes = read_pack.step.load(Ordering::Relaxed);
    149
    150            let needs_percentage_update = last_percentage_update.elapsed() >= slow_check_interval;
    151            if needs_percentage_update {
    152                counter.add(received_bytes, now);
    153                last_percentage_update = now;
    154            }
    155            let rate = HumanBytes(counter.rate() as u64);
    156            let msg = format!(", {rate:.2}/s");
    157
    158            progress_bar.tick(
    159                (total_objects * (num_phases - 2)) + objects,
    160                total_objects * num_phases,
    161                &msg,
    162            )?;
    163        } else if let Some((action, remote)) =
    164            find_in(&tasks, |t| progress_by_id(remote_progress, t))
    165        {
    166            if !is_shallow {
    167                continue;
    168            }
    169            // phase 1: work on the remote side
    170
    171            // Resolving deltas.
    172            let objects = remote.step.load(Ordering::Relaxed);
    173            if let Some(total_objects) = remote.done_at {
    174                let msg = format!(", ({objects}/{total_objects}) {action}");
    175                progress_bar.tick(objects, total_objects * num_phases, &msg)?;
    176            }
    177        }
    178    }
    179    Ok(())
    180}
    181
    182fn amend_authentication_hints(
    183    res: Result<(), crate::sources::git::fetch::Error>,
    184    remote_url: &str,
    185    last_url_for_authentication: Option<gix::bstr::BString>,
    186) -> CargoResult<()> {
    187    let Err(err) = res else { return Ok(()) };
    188    let e = match &err {
    189        crate::sources::git::fetch::Error::PrepareFetch(
    190            gix::remote::fetch::prepare::Error::RefMap(gix::remote::ref_map::Error::Handshake(err)),
    191        ) => Some(err),
    192        _ => None,
    193    };
    194
    195    if let Some(e) = e {
    196        let auth_message = match e {
    197            gix::protocol::handshake::Error::Credentials(_) => {
    198                "\n* attempted to find username/password via \
    199                     git's `credential.helper` support, but failed"
    200                    .into()
    201            }
    202            gix::protocol::handshake::Error::InvalidCredentials { .. } => {
    203                "\n* attempted to find username/password via \
    204                     `credential.helper`, but maybe the found \
    205                     credentials were incorrect"
    206                    .into()
    207            }
    208            gix::protocol::handshake::Error::Transport(_) => {
    209                let msg = format!(
    210                    concat!(
    211                        "network failure seems to have happened\n",
    212                        "if a proxy or similar is necessary `net.git-fetch-with-cli` may help here\n",
    213                        "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli",
    214                        "{}"
    215                    ),
    216                    super::utils::note_github_pull_request(remote_url).unwrap_or_default()
    217                );
    218                return Err(anyhow::Error::from(err).context(msg));
    219            }
    220            _ => None,
    221        };
    222        if let Some(auth_message) = auth_message {
    223            let mut msg = "failed to authenticate when downloading \
    224                       repository"
    225                .to_string();
    226            if let Some(url) = last_url_for_authentication {
    227                msg.push_str(": ");
    228                msg.push_str(url.to_str_lossy().as_ref());
    229            }
    230            msg.push('\n');
    231            msg.push_str(auth_message);
    232            msg.push_str("\n\n");
    233            msg.push_str("if the git CLI succeeds then `net.git-fetch-with-cli` may help here\n");
    234            msg.push_str(
    235                "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli",
    236            );
    237            return Err(anyhow::Error::from(err).context(msg));
    238        }
    239    }
    240    Err(err.into())
    241}
    242
    243/// The reason we are opening a git repository.
    244///
    245/// This can affect the way we open it and the cost associated with it.
    246pub enum OpenMode {
    247    /// We need `git_binary` configuration as well for being able to see credential helpers
    248    /// that are configured with the `git` installation itself.
    249    /// However, this is slow on windows (~150ms) and most people won't need it as they use the
    250    /// standard index which won't ever need authentication, so we only enable this when needed.
    251    ForFetch,
    252}
    253
    254impl OpenMode {
    255    /// Sometimes we don't need to pay for figuring out the system's git installation, and this tells
    256    /// us if that is the case.
    257    pub fn needs_git_binary_config(&self) -> bool {
    258        match self {
    259            OpenMode::ForFetch => true,
    260        }
    261    }
    262}
    263
    264/// Produce a repository with everything pre-configured according to `config`. Most notably this includes
    265/// transport configuration. Knowing its `purpose` helps to optimize the way we open the repository.
    266/// Use `config_overrides` to configure the new repository.
    267pub fn open_repo(
    268    repo_path: &std::path::Path,
    269    config_overrides: Vec<BString>,
    270    purpose: OpenMode,
    271) -> Result<gix::Repository, gix::open::Error> {
    272    gix::open_opts(repo_path, {
    273        let mut opts = gix::open::Options::default();
    274        opts.permissions.config = gix::open::permissions::Config::all();
    275        opts.permissions.config.git_binary = purpose.needs_git_binary_config();
    276        opts.with(gix::sec::Trust::Full)
    277            .config_overrides(config_overrides)
    278    })
    279}
    280
    281/// Convert `git` related cargo configuration into the respective `git` configuration which can be
    282/// used when opening new repositories.
    283pub fn cargo_config_to_gitoxide_overrides(gctx: &GlobalContext) -> CargoResult<Vec<BString>> {
    284    use gix::config::tree::{Core, Http, Key, gitoxide};
    285    let timeout = HttpTimeout::new(gctx)?;
    286    let http = gctx.http_config()?;
    287
    288    let mut values = vec![
    289        gitoxide::Http::CONNECT_TIMEOUT.validated_assignment_fmt(&timeout.dur.as_millis())?,
    290        Http::LOW_SPEED_LIMIT.validated_assignment_fmt(&timeout.low_speed_limit)?,
    291        Http::LOW_SPEED_TIME.validated_assignment_fmt(&timeout.dur.as_secs())?,
    292        // Assure we are not depending on committer information when updating refs after cloning.
    293        Core::LOG_ALL_REF_UPDATES.validated_assignment_fmt(&false)?,
    294    ];
    295    if let Some(proxy) = &http.proxy {
    296        values.push(Http::PROXY.validated_assignment_fmt(proxy)?);
    297    }
    298    if let Some(check_revoke) = http.check_revoke {
    299        values.push(Http::SCHANNEL_CHECK_REVOKE.validated_assignment_fmt(&check_revoke)?);
    300    }
    301    if let Some(cainfo) = &http.cainfo {
    302        values.push(
    303            Http::SSL_CA_INFO.validated_assignment_fmt(&cainfo.resolve_path(gctx).display())?,
    304        );
    305    }
    306
    307    values.push(if let Some(user_agent) = &http.user_agent {
    308        Http::USER_AGENT.validated_assignment_fmt(user_agent)
    309    } else {
    310        Http::USER_AGENT.validated_assignment_fmt(&format!("cargo {}", crate::version()))
    311    }?);
    312    if let Some(ssl_version) = &http.ssl_version {
    313        use crate::util::context::SslVersionConfig;
    314        match ssl_version {
    315            SslVersionConfig::Single(version) => {
    316                values.push(Http::SSL_VERSION.validated_assignment_fmt(&version)?);
    317            }
    318            SslVersionConfig::Range(range) => {
    319                values.push(
    320                    gitoxide::Http::SSL_VERSION_MIN
    321                        .validated_assignment_fmt(&range.min.as_deref().unwrap_or("default"))?,
    322                );
    323                values.push(
    324                    gitoxide::Http::SSL_VERSION_MAX
    325                        .validated_assignment_fmt(&range.max.as_deref().unwrap_or("default"))?,
    326                );
    327            }
    328        }
    329    } else if cfg!(windows) {
    330        // This text is copied from https://github.com/rust-lang/cargo/blob/39c13e67a5962466cc7253d41bc1099bbcb224c3/src/cargo/ops/registry.rs#L658-L674 .
    331        // This is a temporary workaround for some bugs with libcurl and
    332        // schannel and TLS 1.3.
    333        //
    334        // Our libcurl on Windows is usually built with schannel.
    335        // On Windows 11 (or Windows Server 2022), libcurl recently (late
    336        // 2022) gained support for TLS 1.3 with schannel, and it now defaults
    337        // to 1.3. Unfortunately there have been some bugs with this.
    338        // https://github.com/curl/curl/issues/9431 is the most recent. Once
    339        // that has been fixed, and some time has passed where we can be more
    340        // confident that the 1.3 support won't cause issues, this can be
    341        // removed.
    342        //
    343        // Windows 10 is unaffected. libcurl does not support TLS 1.3 on
    344        // Windows 10. (Windows 10 sorta had support, but it required enabling
    345        // an advanced option in the registry which was buggy, and libcurl
    346        // does runtime checks to prevent it.)
    347        values.push(gitoxide::Http::SSL_VERSION_MIN.validated_assignment_fmt(&"default")?);
    348        values.push(gitoxide::Http::SSL_VERSION_MAX.validated_assignment_fmt(&"tlsv1.2")?);
    349    }
    350    if let Some(debug) = http.debug {
    351        values.push(gitoxide::Http::VERBOSE.validated_assignment_fmt(&debug)?);
    352    }
    353    if let Some(multiplexing) = http.multiplexing {
    354        let http_version = multiplexing.then(|| "HTTP/2").unwrap_or("HTTP/1.1");
    355        // Note that failing to set the HTTP version in `gix-transport` isn't fatal,
    356        // which is why we don't have to try to figure out if HTTP V2 is supported in the
    357        // currently linked version (see `try_old_curl!()`)
    358        values.push(Http::VERSION.validated_assignment_fmt(&http_version)?);
    359    }
    360
    361    Ok(values)
    362}
    363
    364/// Reinitializes a given Git repository. This is useful when a Git repository
    365/// seems corrupted, and we want to start over.
    366pub fn reinitialize(git_dir: &Path) -> CargoResult<()> {
    367    fn init(path: &Path, bare: bool) -> CargoResult<()> {
    368        let mut opts = git2::RepositoryInitOptions::new();
    369        // Skip anything related to templates, they just call all sorts of issues as
    370        // we really don't want to use them yet they insist on being used. See #6240
    371        // for an example issue that comes up.
    372        opts.external_template(false);
    373        opts.bare(bare);
    374        git2::Repository::init_opts(&path, &opts)?;
    375        Ok(())
    376    }
    377    // Here we want to drop the current repository object pointed to by `repo`,
    378    // so we initialize temporary repository in a sub-folder, blow away the
    379    // existing git folder, and then recreate the git repo. Finally we blow away
    380    // the `tmp` folder we allocated.
    381    debug!("reinitializing git repo at {:?}", git_dir);
    382    let tmp = git_dir.join("tmp");
    383    let bare = !git_dir.ends_with(".git");
    384    init(&tmp, false)?;
    385    for entry in git_dir.read_dir()? {
    386        let entry = entry?;
    387        if entry.file_name().to_str() == Some("tmp") {
    388            continue;
    389        }
    390        let path = entry.path();
    391        drop(paths::remove_file(&path).or_else(|_| paths::remove_dir_all(&path)));
    392    }
    393    init(git_dir, bare)?;
    394    paths::remove_dir_all(&tmp)?;
    395    Ok(())
    396}