#
cargo/sources/git/
source.rs

    1//! See [`GitSource`].
    2
    3use crate::core::GitReference;
    4use crate::core::SourceId;
    5use crate::core::global_cache_tracker;
    6use crate::core::{Dependency, Package, PackageId};
    7use crate::sources::IndexSummary;
    8use crate::sources::RecursivePathSource;
    9use crate::sources::git::utils::GitDatabase;
    10use crate::sources::git::utils::GitRemote;
    11use crate::sources::git::utils::rev_to_oid;
    12use crate::sources::source::MaybePackage;
    13use crate::sources::source::QueryKind;
    14use crate::sources::source::Source;
    15use crate::util::GlobalContext;
    16use crate::util::cache_lock::CacheLockMode;
    17use crate::util::errors::CargoResult;
    18use crate::util::hex::short_hash;
    19use crate::util::interning::InternedString;
    20use anyhow::Context as _;
    21use cargo_util::paths::exclude_from_backups_and_indexing;
    22use std::fmt::{self, Debug, Formatter};
    23use std::task::Poll;
    24use tracing::trace;
    25use url::Url;
    26
    27/// `GitSource` contains one or more packages gathering from a Git repository.
    28/// Under the hood it uses [`RecursivePathSource`] to discover packages inside the
    29/// repository.
    30///
    31/// ## Filesystem layout
    32///
    33/// During a successful `GitSource` download, at least two Git repositories are
    34/// created: one is the shared Git database of this remote, and the other is the
    35/// Git checkout to a specific revision, which contains the actual files to be
    36/// compiled. Multiple checkouts can be cloned from a single Git database.
    37///
    38/// Those repositories are located at Cargo's Git cache directory
    39/// `$CARGO_HOME/git`. The file tree of the cache directory roughly looks like:
    40///
    41/// ```text
    42/// $CARGO_HOME/git/
    43/// ├── checkouts/
    44/// │  ├── gimli-a0d193bd15a5ed96/
    45/// │  │  ├── 8e73ef0/     # Git short ID for a certain revision
    46/// │  │  ├── a2a4b78/
    47/// │  │  └── e33d1ac/
    48/// │  ├── log-c58e1db3de7c154d-shallow/
    49/// │  │  └── 11eda98/
    50/// └── db/
    51///    ├── gimli-a0d193bd15a5ed96/
    52///    └── log-c58e1db3de7c154d-shallow/
    53/// ```
    54///
    55/// For more on Git cache directory, see ["Cargo Home"] in The Cargo Book.
    56///
    57/// For more on the directory format `<pkg>-<hash>[-shallow]`, see [`ident`]
    58/// and [`ident_shallow`].
    59///
    60/// ## Locked to a revision
    61///
    62/// Once a `GitSource` is fetched, it will resolve to a specific commit revision.
    63/// This is often mentioned as "locked revision" (`locked_rev`) throughout the
    64/// codebase. The revision is written into `Cargo.lock`. This is essential since
    65/// we want to ensure a package can compiles with the same set of files when
    66/// a `Cargo.lock` is present. With the `locked_rev` provided, `GitSource` can
    67/// precisely fetch the same revision from the Git repository.
    68///
    69/// ["Cargo Home"]: https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#directories
    70pub struct GitSource<'gctx> {
    71    /// The git remote which we're going to fetch from.
    72    remote: GitRemote,
    73    /// The revision which a git source is locked to.
    74    ///
    75    /// Expected to always be [`Revision::Locked`] after the Git repository is fetched.
    76    locked_rev: Revision,
    77    /// The unique identifier of this source.
    78    source_id: SourceId,
    79    /// The underlying path source to discover packages inside the Git repository.
    80    ///
    81    /// This gets set to `Some` after the git repo has been checked out
    82    /// (automatically handled via [`GitSource::block_until_ready`]).
    83    path_source: Option<RecursivePathSource<'gctx>>,
    84    /// A short string that uniquely identifies the version of the checkout.
    85    ///
    86    /// This is typically a 7-character string of the OID hash, automatically
    87    /// increasing in size if it is ambiguous.
    88    ///
    89    /// This is set to `Some` after the git repo has been checked out
    90    /// (automatically handled via [`GitSource::block_until_ready`]).
    91    short_id: Option<InternedString>,
    92    /// The identifier of this source for Cargo's Git cache directory.
    93    /// See [`ident`] for more.
    94    ident: InternedString,
    95    gctx: &'gctx GlobalContext,
    96    /// Disables status messages.
    97    quiet: bool,
    98}
    99
    100impl<'gctx> GitSource<'gctx> {
    101    /// Creates a git source for the given [`SourceId`].
    102    pub fn new(source_id: SourceId, gctx: &'gctx GlobalContext) -> CargoResult<GitSource<'gctx>> {
    103        assert!(source_id.is_git(), "id is not git, id={}", source_id);
    104
    105        let remote = GitRemote::new(source_id.url());
    106        // Fallback to git ref from manifest if there is no locked revision.
    107        let locked_rev = source_id
    108            .precise_git_fragment()
    109            .map(|s| Revision::new(s.into()))
    110            .unwrap_or_else(|| source_id.git_reference().unwrap().clone().into());
    111
    112        let ident = ident_shallow(
    113            &source_id,
    114            gctx.cli_unstable()
    115                .git
    116                .map_or(false, |features| features.shallow_deps),
    117        );
    118
    119        let source = GitSource {
    120            remote,
    121            locked_rev,
    122            source_id,
    123            path_source: None,
    124            short_id: None,
    125            ident: ident.into(),
    126            gctx,
    127            quiet: false,
    128        };
    129
    130        Ok(source)
    131    }
    132
    133    /// Gets the remote repository URL.
    134    pub fn url(&self) -> &Url {
    135        self.remote.url()
    136    }
    137
    138    /// Returns the packages discovered by this source. It may fetch the Git
    139    /// repository as well as walk the filesystem if package information
    140    /// haven't yet updated.
    141    pub fn read_packages(&mut self) -> CargoResult<Vec<Package>> {
    142        if self.path_source.is_none() {
    143            self.invalidate_cache();
    144            self.block_until_ready()?;
    145        }
    146        self.path_source.as_mut().unwrap().read_packages()
    147    }
    148
    149    fn mark_used(&self) -> CargoResult<()> {
    150        self.gctx
    151            .deferred_global_last_use()?
    152            .mark_git_checkout_used(global_cache_tracker::GitCheckout {
    153                encoded_git_name: self.ident,
    154                short_name: self.short_id.expect("update before download"),
    155                size: None,
    156            });
    157        Ok(())
    158    }
    159
    160    /// Fetch and return a [`GitDatabase`] with the resolved revision
    161    /// for this source,
    162    ///
    163    /// This won't fetch anything if the required revision is
    164    /// already available locally.
    165    pub(crate) fn fetch_db(&self, is_submodule: bool) -> CargoResult<(GitDatabase, git2::Oid)> {
    166        let db_path = self.gctx.git_db_path().join(&self.ident);
    167        let db_path = db_path.into_path_unlocked();
    168
    169        let db = self.remote.db_at(&db_path).ok();
    170
    171        let (db, actual_rev) = match (&self.locked_rev, db) {
    172            // If we have a locked revision, and we have a preexisting database
    173            // which has that revision, then no update needs to happen.
    174            (Revision::Locked(oid), Some(db)) if db.contains(*oid) => (db, *oid),
    175
    176            // If we're in offline mode, we're not locked, and we have a
    177            // database, then try to resolve our reference with the preexisting
    178            // repository.
    179            (Revision::Deferred(git_ref), Some(db)) if !self.gctx.network_allowed() => {
    180                let offline_flag = self
    181                    .gctx
    182                    .offline_flag()
    183                    .expect("always present when `!network_allowed`");
    184                let rev = db.resolve(&git_ref).with_context(|| {
    185                    format!(
    186                        "failed to lookup reference in preexisting repository, and \
    187                         can't check for updates in offline mode ({offline_flag})"
    188                    )
    189                })?;
    190                (db, rev)
    191            }
    192
    193            // ... otherwise we use this state to update the git database. Note
    194            // that we still check for being offline here, for example in the
    195            // situation that we have a locked revision but the database
    196            // doesn't have it.
    197            (locked_rev, db) => {
    198                if let Some(offline_flag) = self.gctx.offline_flag() {
    199                    anyhow::bail!(
    200                        "can't checkout from '{}': you are in the offline mode ({offline_flag})",
    201                        self.remote.url()
    202                    );
    203                }
    204
    205                if !self.quiet {
    206                    let scope = if is_submodule {
    207                        "submodule"
    208                    } else {
    209                        "repository"
    210                    };
    211                    self.gctx
    212                        .shell()
    213                        .status("Updating", format!("git {scope} `{}`", self.remote.url()))?;
    214                }
    215
    216                trace!("updating git source `{:?}`", self.remote);
    217
    218                let locked_rev = locked_rev.clone().into();
    219                self.remote.checkout(&db_path, db, &locked_rev, self.gctx)?
    220            }
    221        };
    222        Ok((db, actual_rev))
    223    }
    224}
    225
    226/// Indicates a [Git revision] that might be locked or deferred to be resolved.
    227///
    228/// [Git revision]: https://git-scm.com/docs/revisions
    229#[derive(Clone, Debug)]
    230enum Revision {
    231    /// A [Git reference] that would trigger extra fetches when being resolved.
    232    ///
    233    /// [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
    234    Deferred(GitReference),
    235    /// A locked revision of the actual Git commit object ID.
    236    Locked(git2::Oid),
    237}
    238
    239impl Revision {
    240    fn new(rev: &str) -> Revision {
    241        match rev_to_oid(rev) {
    242            Some(oid) => Revision::Locked(oid),
    243            None => Revision::Deferred(GitReference::Rev(rev.to_string())),
    244        }
    245    }
    246}
    247
    248impl From<GitReference> for Revision {
    249    fn from(value: GitReference) -> Self {
    250        Revision::Deferred(value)
    251    }
    252}
    253
    254impl From<Revision> for GitReference {
    255    fn from(value: Revision) -> Self {
    256        match value {
    257            Revision::Deferred(git_ref) => git_ref,
    258            Revision::Locked(oid) => GitReference::Rev(oid.to_string()),
    259        }
    260    }
    261}
    262
    263/// Create an identifier from a URL,
    264/// essentially turning `proto://host/path/repo` into `repo-<hash-of-url>`.
    265fn ident(id: &SourceId) -> String {
    266    let ident = id
    267        .canonical_url()
    268        .raw_canonicalized_url()
    269        .path_segments()
    270        .and_then(|s| s.rev().next())
    271        .unwrap_or("");
    272
    273    let ident = if ident.is_empty() { "_empty" } else { ident };
    274
    275    format!("{}-{}", ident, short_hash(id.canonical_url()))
    276}
    277
    278/// Like [`ident()`], but appends `-shallow` to it, turning
    279/// `proto://host/path/repo` into `repo-<hash-of-url>-shallow`.
    280///
    281/// It's important to separate shallow from non-shallow clones for reasons of
    282/// backwards compatibility --- older cargo's aren't necessarily handling
    283/// shallow clones correctly.
    284fn ident_shallow(id: &SourceId, is_shallow: bool) -> String {
    285    let mut ident = ident(id);
    286    if is_shallow {
    287        ident.push_str("-shallow");
    288    }
    289    ident
    290}
    291
    292impl<'gctx> Debug for GitSource<'gctx> {
    293    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
    294        write!(f, "git repo at {}", self.remote.url())?;
    295        match &self.locked_rev {
    296            Revision::Deferred(git_ref) => match git_ref.pretty_ref(true) {
    297                Some(s) => write!(f, " ({})", s),
    298                None => Ok(()),
    299            },
    300            Revision::Locked(oid) => write!(f, " ({oid})"),
    301        }
    302    }
    303}
    304
    305impl<'gctx> Source for GitSource<'gctx> {
    306    fn query(
    307        &mut self,
    308        dep: &Dependency,
    309        kind: QueryKind,
    310        f: &mut dyn FnMut(IndexSummary),
    311    ) -> Poll<CargoResult<()>> {
    312        if let Some(src) = self.path_source.as_mut() {
    313            src.query(dep, kind, f)
    314        } else {
    315            Poll::Pending
    316        }
    317    }
    318
    319    fn supports_checksums(&self) -> bool {
    320        false
    321    }
    322
    323    fn requires_precise(&self) -> bool {
    324        true
    325    }
    326
    327    fn source_id(&self) -> SourceId {
    328        self.source_id
    329    }
    330
    331    fn block_until_ready(&mut self) -> CargoResult<()> {
    332        if self.path_source.is_some() {
    333            self.mark_used()?;
    334            return Ok(());
    335        }
    336
    337        let git_fs = self.gctx.git_path();
    338        // Ignore errors creating it, in case this is a read-only filesystem:
    339        // perhaps the later operations can succeed anyhow.
    340        let _ = git_fs.create_dir();
    341        let git_path = self
    342            .gctx
    343            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &git_fs);
    344
    345        // Before getting a checkout, make sure that `<cargo_home>/git` is
    346        // marked as excluded from indexing and backups. Older versions of Cargo
    347        // didn't do this, so we do it here regardless of whether `<cargo_home>`
    348        // exists.
    349        //
    350        // This does not use `create_dir_all_excluded_from_backups_atomic` for
    351        // the same reason: we want to exclude it even if the directory already
    352        // exists.
    353        exclude_from_backups_and_indexing(&git_path);
    354
    355        let (db, actual_rev) = self.fetch_db(false)?;
    356
    357        // Don’t use the full hash, in order to contribute less to reaching the
    358        // path length limit on Windows. See
    359        // <https://github.com/servo/servo/pull/14397>.
    360        let short_id = db.to_short_id(actual_rev)?;
    361
    362        // Check out `actual_rev` from the database to a scoped location on the
    363        // filesystem. This will use hard links and such to ideally make the
    364        // checkout operation here pretty fast.
    365        let checkout_path = self
    366            .gctx
    367            .git_checkouts_path()
    368            .join(&self.ident)
    369            .join(short_id.as_str());
    370        let checkout_path = checkout_path.into_path_unlocked();
    371        db.copy_to(actual_rev, &checkout_path, self.gctx, self.quiet)?;
    372
    373        let source_id = self
    374            .source_id
    375            .with_git_precise(Some(actual_rev.to_string()));
    376        let path_source = RecursivePathSource::new(&checkout_path, source_id, self.gctx);
    377
    378        self.path_source = Some(path_source);
    379        self.short_id = Some(short_id.as_str().into());
    380        self.locked_rev = Revision::Locked(actual_rev);
    381        self.path_source.as_mut().unwrap().load()?;
    382
    383        self.mark_used()?;
    384        Ok(())
    385    }
    386
    387    fn download(&mut self, id: PackageId) -> CargoResult<MaybePackage> {
    388        trace!(
    389            "getting packages for package ID `{}` from `{:?}`",
    390            id, self.remote
    391        );
    392        self.mark_used()?;
    393        self.path_source
    394            .as_mut()
    395            .expect("BUG: `update()` must be called before `get()`")
    396            .download(id)
    397    }
    398
    399    fn finish_download(&mut self, _id: PackageId, _data: Vec<u8>) -> CargoResult<Package> {
    400        panic!("no download should have started")
    401    }
    402
    403    fn fingerprint(&self, _pkg: &Package) -> CargoResult<String> {
    404        match &self.locked_rev {
    405            Revision::Locked(oid) => Ok(oid.to_string()),
    406            _ => unreachable!("locked_rev must be resolved when computing fingerprint"),
    407        }
    408    }
    409
    410    fn describe(&self) -> String {
    411        format!("Git repository {}", self.source_id)
    412    }
    413
    414    fn add_to_yanked_whitelist(&mut self, _pkgs: &[PackageId]) {}
    415
    416    fn is_yanked(&mut self, _pkg: PackageId) -> Poll<CargoResult<bool>> {
    417        Poll::Ready(Ok(false))
    418    }
    419
    420    fn invalidate_cache(&mut self) {}
    421
    422    fn set_quiet(&mut self, quiet: bool) {
    423        self.quiet = quiet;
    424    }
    425}
    426
    427#[cfg(test)]
    428mod test {
    429    use super::ident;
    430    use crate::core::{GitReference, SourceId};
    431    use crate::util::IntoUrl;
    432
    433    #[test]
    434    pub fn test_url_to_path_ident_with_path() {
    435        let ident = ident(&src("https://github.com/carlhuda/cargo"));
    436        assert!(ident.starts_with("cargo-"));
    437    }
    438
    439    #[test]
    440    pub fn test_url_to_path_ident_without_path() {
    441        let ident = ident(&src("https://github.com"));
    442        assert!(ident.starts_with("_empty-"));
    443    }
    444
    445    #[test]
    446    fn test_canonicalize_idents_by_stripping_trailing_url_slash() {
    447        let ident1 = ident(&src("https://github.com/PistonDevelopers/piston/"));
    448        let ident2 = ident(&src("https://github.com/PistonDevelopers/piston"));
    449        assert_eq!(ident1, ident2);
    450    }
    451
    452    #[test]
    453    fn test_canonicalize_idents_by_lowercasing_github_urls() {
    454        let ident1 = ident(&src("https://github.com/PistonDevelopers/piston"));
    455        let ident2 = ident(&src("https://github.com/pistondevelopers/piston"));
    456        assert_eq!(ident1, ident2);
    457    }
    458
    459    #[test]
    460    fn test_canonicalize_idents_by_stripping_dot_git() {
    461        let ident1 = ident(&src("https://github.com/PistonDevelopers/piston"));
    462        let ident2 = ident(&src("https://github.com/PistonDevelopers/piston.git"));
    463        assert_eq!(ident1, ident2);
    464    }
    465
    466    #[test]
    467    fn test_canonicalize_idents_different_protocols() {
    468        let ident1 = ident(&src("https://github.com/PistonDevelopers/piston"));
    469        let ident2 = ident(&src("git://github.com/PistonDevelopers/piston"));
    470        assert_eq!(ident1, ident2);
    471    }
    472
    473    fn src(s: &str) -> SourceId {
    474        SourceId::for_git(&s.into_url().unwrap(), GitReference::DefaultBranch).unwrap()
    475    }
    476}