#
cargo/sources/git/
oxide.rs

    1//! This module contains all code sporting `gitoxide` for operations on `git` repositories and it mirrors
    2//! `utils` closely for now. One day it can be renamed into `utils` once `git2` isn't required anymore.
    3
    4use crate::util::HumanBytes;
    5use crate::util::network::http::HttpTimeout;
    6use crate::util::{MetricsCounter, Progress, network};
    7use crate::{CargoResult, GlobalContext};
    8use cargo_util::paths;
    9use gix::bstr::{BString, ByteSlice};
    10use std::cell::RefCell;
    11use std::path::Path;
    12use std::sync::atomic::{AtomicBool, Ordering};
    13use std::sync::{Arc, Weak};
    14use std::time::{Duration, Instant};
    15use tracing::debug;
    16
    17/// For the time being, `repo_path` makes it easy to instantiate a gitoxide repo just for fetching.
    18/// In future this may change to be the gitoxide repository itself.
    19pub fn with_retry_and_progress(
    20    repo_path: &std::path::Path,
    21    gctx: &GlobalContext,
    22    cb: &(
    23         dyn Fn(
    24        &std::path::Path,
    25        &AtomicBool,
    26        &mut gix::progress::tree::Item,
    27        &mut dyn FnMut(&gix::bstr::BStr),
    28    ) -> Result<(), crate::sources::git::fetch::Error>
    29             + Send
    30             + Sync
    31     ),
    32) -> CargoResult<()> {
    33    std::thread::scope(|s| {
    34        let mut progress_bar = Progress::new("Fetch", gctx);
    35        let is_shallow = gctx.cli_unstable().git.map_or(false, |features| {
    36            features.shallow_deps || features.shallow_index
    37        });
    38        network::retry::with_retry(gctx, || {
    39            let progress_root: Arc<gix::progress::tree::Root> =
    40                gix::progress::tree::root::Options {
    41                    initial_capacity: 10,
    42                    message_buffer_capacity: 10,
    43                }
    44                .into();
    45            let root = Arc::downgrade(&progress_root);
    46            let thread = s.spawn(move || {
    47                let mut progress = progress_root.add_child("operation");
    48                let mut urls = RefCell::new(Default::default());
    49                let res = cb(
    50                    &repo_path,
    51                    &AtomicBool::default(),
    52                    &mut progress,
    53                    &mut |url| {
    54                        *urls.borrow_mut() = Some(url.to_owned());
    55                    },
    56                );
    57                amend_authentication_hints(res, urls.get_mut().take())
    58            });
    59            translate_progress_to_bar(&mut progress_bar, root, is_shallow)?;
    60            thread.join().expect("no panic in scoped thread")
    61        })
    62    })
    63}
    64
    65fn translate_progress_to_bar(
    66    progress_bar: &mut Progress<'_>,
    67    root: Weak<gix::progress::tree::Root>,
    68    is_shallow: bool,
    69) -> CargoResult<()> {
    70    let remote_progress: gix::progress::Id = gix::remote::fetch::ProgressId::RemoteProgress.into();
    71    let read_pack_bytes: gix::progress::Id =
    72        gix::odb::pack::bundle::write::ProgressId::ReadPackBytes.into();
    73    let delta_index_objects: gix::progress::Id =
    74        gix::odb::pack::index::write::ProgressId::IndexObjects.into();
    75    let resolve_objects: gix::progress::Id =
    76        gix::odb::pack::index::write::ProgressId::ResolveObjects.into();
    77
    78    // We choose `N=10` here to make a `300ms * 10slots ~= 3000ms`
    79    // sliding window for tracking the data transfer rate (in bytes/s).
    80    let mut last_percentage_update = Instant::now();
    81    let mut last_fast_update = Instant::now();
    82    let mut counter = MetricsCounter::<10>::new(0, last_percentage_update);
    83
    84    let mut tasks = Vec::with_capacity(10);
    85    let slow_check_interval = std::time::Duration::from_millis(300);
    86    let fast_check_interval = Duration::from_millis(50);
    87    let sleep_interval = Duration::from_millis(10);
    88    debug_assert_eq!(
    89        slow_check_interval.as_millis() % fast_check_interval.as_millis(),
    90        0,
    91        "progress should be smoother by keeping these as multiples of each other"
    92    );
    93    debug_assert_eq!(
    94        fast_check_interval.as_millis() % sleep_interval.as_millis(),
    95        0,
    96        "progress should be smoother by keeping these as multiples of each other"
    97    );
    98
    99    let num_phases = if is_shallow { 3 } else { 2 }; // indexing + delta-resolution, both with same amount of objects to handle
    100    while let Some(root) = root.upgrade() {
    101        std::thread::sleep(sleep_interval);
    102        let needs_update = last_fast_update.elapsed() >= fast_check_interval;
    103        if !needs_update {
    104            continue;
    105        }
    106        let now = Instant::now();
    107        last_fast_update = now;
    108
    109        root.sorted_snapshot(&mut tasks);
    110
    111        fn progress_by_id(
    112            id: gix::progress::Id,
    113            task: &gix::progress::Task,
    114        ) -> Option<(&str, &gix::progress::Value)> {
    115            (task.id == id)
    116                .then(|| task.progress.as_ref())
    117                .flatten()
    118                .map(|value| (task.name.as_str(), value))
    119        }
    120        fn find_in<K>(
    121            tasks: &[(K, gix::progress::Task)],
    122            cb: impl Fn(&gix::progress::Task) -> Option<(&str, &gix::progress::Value)>,
    123        ) -> Option<(&str, &gix::progress::Value)> {
    124            tasks.iter().find_map(|(_, t)| cb(t))
    125        }
    126
    127        if let Some((_, objs)) = find_in(&tasks, |t| progress_by_id(resolve_objects, t)) {
    128            // Phase 3: Resolving deltas.
    129            let objects = objs.step.load(Ordering::Relaxed);
    130            let total_objects = objs.done_at.expect("known amount of objects");
    131            let msg = format!(", ({objects}/{total_objects}) resolving deltas");
    132
    133            progress_bar.tick(
    134                (total_objects * (num_phases - 1)) + objects,
    135                total_objects * num_phases,
    136                &msg,
    137            )?;
    138        } else if let Some((objs, read_pack)) =
    139            find_in(&tasks, |t| progress_by_id(read_pack_bytes, t)).and_then(|read| {
    140                find_in(&tasks, |t| progress_by_id(delta_index_objects, t))
    141                    .map(|delta| (delta.1, read.1))
    142            })
    143        {
    144            // Phase 2: Receiving objects.
    145            let objects = objs.step.load(Ordering::Relaxed);
    146            let total_objects = objs.done_at.expect("known amount of objects");
    147            let received_bytes = read_pack.step.load(Ordering::Relaxed);
    148
    149            let needs_percentage_update = last_percentage_update.elapsed() >= slow_check_interval;
    150            if needs_percentage_update {
    151                counter.add(received_bytes, now);
    152                last_percentage_update = now;
    153            }
    154            let rate = HumanBytes(counter.rate() as u64);
    155            let msg = format!(", {rate:.2}/s");
    156
    157            progress_bar.tick(
    158                (total_objects * (num_phases - 2)) + objects,
    159                total_objects * num_phases,
    160                &msg,
    161            )?;
    162        } else if let Some((action, remote)) =
    163            find_in(&tasks, |t| progress_by_id(remote_progress, t))
    164        {
    165            if !is_shallow {
    166                continue;
    167            }
    168            // phase 1: work on the remote side
    169
    170            // Resolving deltas.
    171            let objects = remote.step.load(Ordering::Relaxed);
    172            if let Some(total_objects) = remote.done_at {
    173                let msg = format!(", ({objects}/{total_objects}) {action}");
    174                progress_bar.tick(objects, total_objects * num_phases, &msg)?;
    175            }
    176        }
    177    }
    178    Ok(())
    179}
    180
    181fn amend_authentication_hints(
    182    res: Result<(), crate::sources::git::fetch::Error>,
    183    last_url_for_authentication: Option<gix::bstr::BString>,
    184) -> CargoResult<()> {
    185    let Err(err) = res else { return Ok(()) };
    186    let e = match &err {
    187        crate::sources::git::fetch::Error::PrepareFetch(
    188            gix::remote::fetch::prepare::Error::RefMap(gix::remote::ref_map::Error::Handshake(err)),
    189        ) => Some(err),
    190        _ => None,
    191    };
    192    if let Some(e) = e {
    193        let auth_message = match e {
    194            gix::protocol::handshake::Error::Credentials(_) => {
    195                "\n* attempted to find username/password via \
    196                     git's `credential.helper` support, but failed"
    197                    .into()
    198            }
    199            gix::protocol::handshake::Error::InvalidCredentials { .. } => {
    200                "\n* attempted to find username/password via \
    201                     `credential.helper`, but maybe the found \
    202                     credentials were incorrect"
    203                    .into()
    204            }
    205            gix::protocol::handshake::Error::Transport(_) => {
    206                let msg = concat!(
    207                    "network failure seems to have happened\n",
    208                    "if a proxy or similar is necessary `net.git-fetch-with-cli` may help here\n",
    209                    "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli"
    210                );
    211                return Err(anyhow::Error::from(err).context(msg));
    212            }
    213            _ => None,
    214        };
    215        if let Some(auth_message) = auth_message {
    216            let mut msg = "failed to authenticate when downloading \
    217                       repository"
    218                .to_string();
    219            if let Some(url) = last_url_for_authentication {
    220                msg.push_str(": ");
    221                msg.push_str(url.to_str_lossy().as_ref());
    222            }
    223            msg.push('\n');
    224            msg.push_str(auth_message);
    225            msg.push_str("\n\n");
    226            msg.push_str("if the git CLI succeeds then `net.git-fetch-with-cli` may help here\n");
    227            msg.push_str(
    228                "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli",
    229            );
    230            return Err(anyhow::Error::from(err).context(msg));
    231        }
    232    }
    233    Err(err.into())
    234}
    235
    236/// The reason we are opening a git repository.
    237///
    238/// This can affect the way we open it and the cost associated with it.
    239pub enum OpenMode {
    240    /// We need `git_binary` configuration as well for being able to see credential helpers
    241    /// that are configured with the `git` installation itself.
    242    /// However, this is slow on windows (~150ms) and most people won't need it as they use the
    243    /// standard index which won't ever need authentication, so we only enable this when needed.
    244    ForFetch,
    245}
    246
    247impl OpenMode {
    248    /// Sometimes we don't need to pay for figuring out the system's git installation, and this tells
    249    /// us if that is the case.
    250    pub fn needs_git_binary_config(&self) -> bool {
    251        match self {
    252            OpenMode::ForFetch => true,
    253        }
    254    }
    255}
    256
    257/// Produce a repository with everything pre-configured according to `config`. Most notably this includes
    258/// transport configuration. Knowing its `purpose` helps to optimize the way we open the repository.
    259/// Use `config_overrides` to configure the new repository.
    260pub fn open_repo(
    261    repo_path: &std::path::Path,
    262    config_overrides: Vec<BString>,
    263    purpose: OpenMode,
    264) -> Result<gix::Repository, gix::open::Error> {
    265    gix::open_opts(repo_path, {
    266        let mut opts = gix::open::Options::default();
    267        opts.permissions.config = gix::open::permissions::Config::all();
    268        opts.permissions.config.git_binary = purpose.needs_git_binary_config();
    269        opts.with(gix::sec::Trust::Full)
    270            .config_overrides(config_overrides)
    271    })
    272}
    273
    274/// Convert `git` related cargo configuration into the respective `git` configuration which can be
    275/// used when opening new repositories.
    276pub fn cargo_config_to_gitoxide_overrides(gctx: &GlobalContext) -> CargoResult<Vec<BString>> {
    277    use gix::config::tree::{Core, Http, Key, gitoxide};
    278    let timeout = HttpTimeout::new(gctx)?;
    279    let http = gctx.http_config()?;
    280
    281    let mut values = vec![
    282        gitoxide::Http::CONNECT_TIMEOUT.validated_assignment_fmt(&timeout.dur.as_millis())?,
    283        Http::LOW_SPEED_LIMIT.validated_assignment_fmt(&timeout.low_speed_limit)?,
    284        Http::LOW_SPEED_TIME.validated_assignment_fmt(&timeout.dur.as_secs())?,
    285        // Assure we are not depending on committer information when updating refs after cloning.
    286        Core::LOG_ALL_REF_UPDATES.validated_assignment_fmt(&false)?,
    287    ];
    288    if let Some(proxy) = &http.proxy {
    289        values.push(Http::PROXY.validated_assignment_fmt(proxy)?);
    290    }
    291    if let Some(check_revoke) = http.check_revoke {
    292        values.push(Http::SCHANNEL_CHECK_REVOKE.validated_assignment_fmt(&check_revoke)?);
    293    }
    294    if let Some(cainfo) = &http.cainfo {
    295        values.push(
    296            Http::SSL_CA_INFO.validated_assignment_fmt(&cainfo.resolve_path(gctx).display())?,
    297        );
    298    }
    299
    300    values.push(if let Some(user_agent) = &http.user_agent {
    301        Http::USER_AGENT.validated_assignment_fmt(user_agent)
    302    } else {
    303        Http::USER_AGENT.validated_assignment_fmt(&format!("cargo {}", crate::version()))
    304    }?);
    305    if let Some(ssl_version) = &http.ssl_version {
    306        use crate::util::context::SslVersionConfig;
    307        match ssl_version {
    308            SslVersionConfig::Single(version) => {
    309                values.push(Http::SSL_VERSION.validated_assignment_fmt(&version)?);
    310            }
    311            SslVersionConfig::Range(range) => {
    312                values.push(
    313                    gitoxide::Http::SSL_VERSION_MIN
    314                        .validated_assignment_fmt(&range.min.as_deref().unwrap_or("default"))?,
    315                );
    316                values.push(
    317                    gitoxide::Http::SSL_VERSION_MAX
    318                        .validated_assignment_fmt(&range.max.as_deref().unwrap_or("default"))?,
    319                );
    320            }
    321        }
    322    } else if cfg!(windows) {
    323        // This text is copied from https://github.com/rust-lang/cargo/blob/39c13e67a5962466cc7253d41bc1099bbcb224c3/src/cargo/ops/registry.rs#L658-L674 .
    324        // This is a temporary workaround for some bugs with libcurl and
    325        // schannel and TLS 1.3.
    326        //
    327        // Our libcurl on Windows is usually built with schannel.
    328        // On Windows 11 (or Windows Server 2022), libcurl recently (late
    329        // 2022) gained support for TLS 1.3 with schannel, and it now defaults
    330        // to 1.3. Unfortunately there have been some bugs with this.
    331        // https://github.com/curl/curl/issues/9431 is the most recent. Once
    332        // that has been fixed, and some time has passed where we can be more
    333        // confident that the 1.3 support won't cause issues, this can be
    334        // removed.
    335        //
    336        // Windows 10 is unaffected. libcurl does not support TLS 1.3 on
    337        // Windows 10. (Windows 10 sorta had support, but it required enabling
    338        // an advanced option in the registry which was buggy, and libcurl
    339        // does runtime checks to prevent it.)
    340        values.push(gitoxide::Http::SSL_VERSION_MIN.validated_assignment_fmt(&"default")?);
    341        values.push(gitoxide::Http::SSL_VERSION_MAX.validated_assignment_fmt(&"tlsv1.2")?);
    342    }
    343    if let Some(debug) = http.debug {
    344        values.push(gitoxide::Http::VERBOSE.validated_assignment_fmt(&debug)?);
    345    }
    346    if let Some(multiplexing) = http.multiplexing {
    347        let http_version = multiplexing.then(|| "HTTP/2").unwrap_or("HTTP/1.1");
    348        // Note that failing to set the HTTP version in `gix-transport` isn't fatal,
    349        // which is why we don't have to try to figure out if HTTP V2 is supported in the
    350        // currently linked version (see `try_old_curl!()`)
    351        values.push(Http::VERSION.validated_assignment_fmt(&http_version)?);
    352    }
    353
    354    Ok(values)
    355}
    356
    357/// Reinitializes a given Git repository. This is useful when a Git repository
    358/// seems corrupted, and we want to start over.
    359pub fn reinitialize(git_dir: &Path) -> CargoResult<()> {
    360    fn init(path: &Path, bare: bool) -> CargoResult<()> {
    361        let mut opts = git2::RepositoryInitOptions::new();
    362        // Skip anything related to templates, they just call all sorts of issues as
    363        // we really don't want to use them yet they insist on being used. See #6240
    364        // for an example issue that comes up.
    365        opts.external_template(false);
    366        opts.bare(bare);
    367        git2::Repository::init_opts(&path, &opts)?;
    368        Ok(())
    369    }
    370    // Here we want to drop the current repository object pointed to by `repo`,
    371    // so we initialize temporary repository in a sub-folder, blow away the
    372    // existing git folder, and then recreate the git repo. Finally we blow away
    373    // the `tmp` folder we allocated.
    374    debug!("reinitializing git repo at {:?}", git_dir);
    375    let tmp = git_dir.join("tmp");
    376    let bare = !git_dir.ends_with(".git");
    377    init(&tmp, false)?;
    378    for entry in git_dir.read_dir()? {
    379        let entry = entry?;
    380        if entry.file_name().to_str() == Some("tmp") {
    381            continue;
    382        }
    383        let path = entry.path();
    384        drop(paths::remove_file(&path).or_else(|_| paths::remove_dir_all(&path)));
    385    }
    386    init(git_dir, bare)?;
    387    paths::remove_dir_all(&tmp)?;
    388    Ok(())
    389}