#
cargo/sources/registry/
remote.rs

    1//! Access to a Git index based registry. See [`RemoteRegistry`] for details.
    2
    3use crate::core::global_cache_tracker;
    4use crate::core::{GitReference, PackageId, SourceId};
    5use crate::sources::git;
    6use crate::sources::git::fetch::RemoteKind;
    7use crate::sources::git::resolve_ref;
    8use crate::sources::registry::download;
    9use crate::sources::registry::MaybeLock;
    10use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
    11use crate::util::cache_lock::CacheLockMode;
    12use crate::util::errors::CargoResult;
    13use crate::util::interning::InternedString;
    14use crate::util::{Filesystem, GlobalContext};
    15use anyhow::Context as _;
    16use cargo_util::paths;
    17use lazycell::LazyCell;
    18use std::cell::{Cell, Ref, RefCell};
    19use std::fs::File;
    20use std::mem;
    21use std::path::Path;
    22use std::str;
    23use std::task::{ready, Poll};
    24use tracing::{debug, trace};
    25
    26/// A remote registry is a registry that lives at a remote URL (such as
    27/// crates.io). The git index is cloned locally, and `.crate` files are
    28/// downloaded as needed and cached locally.
    29///
    30/// This type is primarily accessed through the [`RegistryData`] trait.
    31///
    32/// See the [module-level documentation](super) for the index format and layout.
    33///
    34/// ## History of Git-based index registry
    35///
    36/// Using Git to host this index used to be quite efficient. The full index can
    37/// be stored efficiently locally on disk, and once it is downloaded, all
    38/// queries of a registry can happen locally and needn't touch the network.
    39/// Git-based index was a reasonable design choice at the time when HTTP/2
    40/// was just introduced.
    41///
    42/// However, the full index keeps growing as crates.io grows. It becomes
    43/// relatively big and slows down the first use of Cargo. Git (specifically
    44/// libgit2) is not efficient at handling huge amounts of small files either.
    45/// On the other hand, newer protocols like HTTP/2 are prevalent and capable to
    46/// serve a bunch of tiny files. Today, it is encouraged to use [`HttpRegistry`],
    47/// which is the default from 1.70.0. That being said, Cargo will continue
    48/// supporting Git-based index for a pretty long while.
    49///
    50/// [`HttpRegistry`]: super::http_remote::HttpRegistry
    51pub struct RemoteRegistry<'gctx> {
    52    /// The name of this source, a unique string (across all sources) used as
    53    /// the directory name where its cached content is stored.
    54    name: InternedString,
    55    /// Path to the registry index (`$CARGO_HOME/registry/index/$REG-HASH`).
    56    index_path: Filesystem,
    57    /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/cache/$REG-HASH`).
    58    cache_path: Filesystem,
    59    /// The unique identifier of this registry source.
    60    source_id: SourceId,
    61    /// This reference is stored so that when a registry needs update, it knows
    62    /// where to fetch from.
    63    index_git_ref: GitReference,
    64    gctx: &'gctx GlobalContext,
    65    /// A Git [tree object] to help this registry find crate metadata from the
    66    /// underlying Git repository.
    67    ///
    68    /// This is stored here to prevent Git from repeatedly creating a tree object
    69    /// during each call into `load()`.
    70    ///
    71    /// [tree object]: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects#_tree_objects
    72    tree: RefCell<Option<git2::Tree<'static>>>,
    73    /// A Git repository that contains the actual index we want.
    74    repo: LazyCell<git2::Repository>,
    75    /// The current HEAD commit of the underlying Git repository.
    76    head: Cell<Option<git2::Oid>>,
    77    /// This stores sha value of the current HEAD commit for convenience.
    78    current_sha: Cell<Option<InternedString>>,
    79    /// Whether this registry needs to update package information.
    80    ///
    81    /// See [`RemoteRegistry::mark_updated`] on how to make sure a registry
    82    /// index is updated only once per session.
    83    needs_update: bool,
    84    /// Disables status messages.
    85    quiet: bool,
    86}
    87
    88impl<'gctx> RemoteRegistry<'gctx> {
    89    /// Creates a Git-rebased remote registry for `source_id`.
    90    ///
    91    /// * `name` --- Name of a path segment where `.crate` tarballs and the
    92    ///   registry index are stored. Expect to be unique.
    93    pub fn new(
    94        source_id: SourceId,
    95        gctx: &'gctx GlobalContext,
    96        name: &str,
    97    ) -> RemoteRegistry<'gctx> {
    98        RemoteRegistry {
    99            name: name.into(),
    100            index_path: gctx.registry_index_path().join(name),
    101            cache_path: gctx.registry_cache_path().join(name),
    102            source_id,
    103            gctx,
    104            index_git_ref: GitReference::DefaultBranch,
    105            tree: RefCell::new(None),
    106            repo: LazyCell::new(),
    107            head: Cell::new(None),
    108            current_sha: Cell::new(None),
    109            needs_update: false,
    110            quiet: false,
    111        }
    112    }
    113
    114    /// Creates intermediate dirs and initialize the repository.
    115    fn repo(&self) -> CargoResult<&git2::Repository> {
    116        self.repo.try_borrow_with(|| {
    117            trace!("acquiring registry index lock");
    118            let path = self
    119                .gctx
    120                .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path);
    121
    122            match git2::Repository::open(&path) {
    123                Ok(repo) => Ok(repo),
    124                Err(_) => {
    125                    drop(paths::remove_dir_all(&path));
    126                    paths::create_dir_all(&path)?;
    127
    128                    // Note that we'd actually prefer to use a bare repository
    129                    // here as we're not actually going to check anything out.
    130                    // All versions of Cargo, though, share the same CARGO_HOME,
    131                    // so for compatibility with older Cargo which *does* do
    132                    // checkouts we make sure to initialize a new full
    133                    // repository (not a bare one).
    134                    //
    135                    // We should change this to `init_bare` whenever we feel
    136                    // like enough time has passed or if we change the directory
    137                    // that the folder is located in, such as by changing the
    138                    // hash at the end of the directory.
    139                    //
    140                    // Note that in the meantime we also skip `init.templatedir`
    141                    // as it can be misconfigured sometimes or otherwise add
    142                    // things that we don't want.
    143                    let mut opts = git2::RepositoryInitOptions::new();
    144                    opts.external_template(false);
    145                    Ok(git2::Repository::init_opts(&path, &opts).with_context(|| {
    146                        format!("failed to initialize index git repository (in {:?})", path)
    147                    })?)
    148                }
    149            }
    150        })
    151    }
    152
    153    /// Get the object ID of the HEAD commit from the underlying Git repository.
    154    fn head(&self) -> CargoResult<git2::Oid> {
    155        if self.head.get().is_none() {
    156            let repo = self.repo()?;
    157            let oid = resolve_ref(&self.index_git_ref, repo)?;
    158            self.head.set(Some(oid));
    159        }
    160        Ok(self.head.get().unwrap())
    161    }
    162
    163    /// Returns a [`git2::Tree`] object of the current HEAD commit of the
    164    /// underlying Git repository.
    165    fn tree(&self) -> CargoResult<Ref<'_, git2::Tree<'_>>> {
    166        {
    167            let tree = self.tree.borrow();
    168            if tree.is_some() {
    169                return Ok(Ref::map(tree, |s| s.as_ref().unwrap()));
    170            }
    171        }
    172        let repo = self.repo()?;
    173        let commit = repo.find_commit(self.head()?)?;
    174        let tree = commit.tree()?;
    175
    176        // SAFETY:
    177        // Unfortunately in libgit2 the tree objects look like they've got a
    178        // reference to the repository object which means that a tree cannot
    179        // outlive the repository that it came from. Here we want to cache this
    180        // tree, though, so to accomplish this we transmute it to a static
    181        // lifetime.
    182        //
    183        // Note that we don't actually hand out the static lifetime, instead we
    184        // only return a scoped one from this function. Additionally the repo
    185        // we loaded from (above) lives as long as this object
    186        // (`RemoteRegistry`) so we then just need to ensure that the tree is
    187        // destroyed first in the destructor, hence the destructor on
    188        // `RemoteRegistry` below.
    189        let tree = unsafe { mem::transmute::<git2::Tree<'_>, git2::Tree<'static>>(tree) };
    190        *self.tree.borrow_mut() = Some(tree);
    191        Ok(Ref::map(self.tree.borrow(), |s| s.as_ref().unwrap()))
    192    }
    193
    194    /// Gets the current version of the registry index.
    195    ///
    196    /// It is usually sha of the HEAD commit from the underlying Git repository.
    197    fn current_version(&self) -> Option<InternedString> {
    198        if let Some(sha) = self.current_sha.get() {
    199            return Some(sha);
    200        }
    201        let sha = self.head().ok()?.to_string().into();
    202        self.current_sha.set(Some(sha));
    203        Some(sha)
    204    }
    205
    206    /// Whether the registry is up-to-date. See [`Self::mark_updated`] for more.
    207    fn is_updated(&self) -> bool {
    208        self.gctx.updated_sources().contains(&self.source_id)
    209    }
    210
    211    /// Marks this registry as up-to-date.
    212    ///
    213    /// This makes sure the index is only updated once per session since it is
    214    /// an expensive operation. This generally only happens when the resolver
    215    /// is run multiple times, such as during `cargo publish`.
    216    fn mark_updated(&self) {
    217        self.gctx.updated_sources().insert(self.source_id);
    218    }
    219}
    220
    221impl<'gctx> RegistryData for RemoteRegistry<'gctx> {
    222    fn prepare(&self) -> CargoResult<()> {
    223        self.repo()?;
    224        self.gctx
    225            .deferred_global_last_use()?
    226            .mark_registry_index_used(global_cache_tracker::RegistryIndex {
    227                encoded_registry_name: self.name,
    228            });
    229        Ok(())
    230    }
    231
    232    fn index_path(&self) -> &Filesystem {
    233        &self.index_path
    234    }
    235
    236    fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path {
    237        self.gctx
    238            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, path)
    239    }
    240
    241    /// Read the general concept for `load()` on [`RegistryData::load`].
    242    ///
    243    /// `index_version` is a string representing the version of the file used
    244    /// to construct the cached copy.
    245    ///
    246    /// Older versions of Cargo used the single value of the hash of the HEAD
    247    /// commit as a `index_version`. This is technically correct but a little
    248    /// too conservative. If a new commit is fetched all cached files need to
    249    /// be regenerated even if a particular file was not changed.
    250    ///
    251    /// However if an old cargo has written such a file we still know how to
    252    /// read it, as long as we check for that hash value.
    253    ///
    254    /// Cargo now uses a hash of the file's contents as provided by git.
    255    fn load(
    256        &mut self,
    257        _root: &Path,
    258        path: &Path,
    259        index_version: Option<&str>,
    260    ) -> Poll<CargoResult<LoadResponse>> {
    261        if self.needs_update {
    262            return Poll::Pending;
    263        }
    264        // Check if the cache is valid.
    265        let git_commit_hash = self.current_version();
    266        if index_version.is_some() && index_version == git_commit_hash.as_deref() {
    267            // This file was written by an old version of cargo, but it is
    268            // still up-to-date.
    269            return Poll::Ready(Ok(LoadResponse::CacheValid));
    270        }
    271        // Note that the index calls this method and the filesystem is locked
    272        // in the index, so we don't need to worry about an `update_index`
    273        // happening in a different process.
    274        fn load_helper(
    275            registry: &RemoteRegistry<'_>,
    276            path: &Path,
    277            index_version: Option<&str>,
    278        ) -> CargoResult<LoadResponse> {
    279            let repo = registry.repo()?;
    280            let tree = registry.tree()?;
    281            let entry = tree.get_path(path);
    282            let entry = entry?;
    283            let git_file_hash = Some(entry.id().to_string());
    284
    285            // Check if the cache is valid.
    286            if index_version.is_some() && index_version == git_file_hash.as_deref() {
    287                return Ok(LoadResponse::CacheValid);
    288            }
    289
    290            let object = entry.to_object(repo)?;
    291            let Some(blob) = object.as_blob() else {
    292                anyhow::bail!("path `{}` is not a blob in the git repo", path.display())
    293            };
    294
    295            Ok(LoadResponse::Data {
    296                raw_data: blob.content().to_vec(),
    297                index_version: git_file_hash,
    298            })
    299        }
    300
    301        match load_helper(&self, path, index_version) {
    302            Ok(result) => Poll::Ready(Ok(result)),
    303            Err(_) if !self.is_updated() => {
    304                // If git returns an error and we haven't updated the repo,
    305                // return pending to allow an update to try again.
    306                self.needs_update = true;
    307                Poll::Pending
    308            }
    309            Err(e)
    310                if e.downcast_ref::<git2::Error>()
    311                    .map(|e| e.code() == git2::ErrorCode::NotFound)
    312                    .unwrap_or_default() =>
    313            {
    314                // The repo has been updated and the file does not exist.
    315                Poll::Ready(Ok(LoadResponse::NotFound))
    316            }
    317            Err(e) => Poll::Ready(Err(e)),
    318        }
    319    }
    320
    321    fn config(&mut self) -> Poll<CargoResult<Option<RegistryConfig>>> {
    322        debug!("loading config");
    323        self.prepare()?;
    324        self.gctx
    325            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path);
    326        match ready!(self.load(Path::new(""), Path::new(RegistryConfig::NAME), None)?) {
    327            LoadResponse::Data { raw_data, .. } => {
    328                trace!("config loaded");
    329                let cfg: RegistryConfig = serde_json::from_slice(&raw_data)?;
    330                Poll::Ready(Ok(Some(cfg)))
    331            }
    332            _ => Poll::Ready(Ok(None)),
    333        }
    334    }
    335
    336    fn block_until_ready(&mut self) -> CargoResult<()> {
    337        if !self.needs_update {
    338            return Ok(());
    339        }
    340
    341        self.needs_update = false;
    342
    343        if self.is_updated() {
    344            return Ok(());
    345        }
    346        self.mark_updated();
    347
    348        if !self.gctx.network_allowed() {
    349            return Ok(());
    350        }
    351        if self.gctx.cli_unstable().no_index_update {
    352            return Ok(());
    353        }
    354
    355        debug!("updating the index");
    356
    357        // Ensure that we'll actually be able to acquire an HTTP handle later on
    358        // once we start trying to download crates. This will weed out any
    359        // problems with `.cargo/config` configuration related to HTTP.
    360        //
    361        // This way if there's a problem the error gets printed before we even
    362        // hit the index, which may not actually read this configuration.
    363        self.gctx.http()?;
    364
    365        self.prepare()?;
    366        self.head.set(None);
    367        *self.tree.borrow_mut() = None;
    368        self.current_sha.set(None);
    369        let _path = self
    370            .gctx
    371            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path);
    372        if !self.quiet {
    373            self.gctx
    374                .shell()
    375                .status("Updating", self.source_id.display_index())?;
    376        }
    377
    378        // Fetch the latest version of our `index_git_ref` into the index
    379        // checkout.
    380        let url = self.source_id.url();
    381        let repo = self.repo.borrow_mut().unwrap();
    382        git::fetch(
    383            repo,
    384            url.as_str(),
    385            &self.index_git_ref,
    386            self.gctx,
    387            RemoteKind::Registry,
    388        )
    389        .with_context(|| format!("failed to fetch `{}`", url))?;
    390
    391        Ok(())
    392    }
    393
    394    /// Read the general concept for `invalidate_cache()` on
    395    /// [`RegistryData::invalidate_cache`].
    396    ///
    397    /// To fully invalidate, undo [`RemoteRegistry::mark_updated`]'s work.
    398    fn invalidate_cache(&mut self) {
    399        self.needs_update = true;
    400    }
    401
    402    fn set_quiet(&mut self, quiet: bool) {
    403        self.quiet = quiet;
    404    }
    405
    406    fn is_updated(&self) -> bool {
    407        self.is_updated()
    408    }
    409
    410    fn download(&mut self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock> {
    411        let registry_config = loop {
    412            match self.config()? {
    413                Poll::Pending => self.block_until_ready()?,
    414                Poll::Ready(cfg) => break cfg.unwrap(),
    415            }
    416        };
    417
    418        download::download(
    419            &self.cache_path,
    420            &self.gctx,
    421            self.name,
    422            pkg,
    423            checksum,
    424            registry_config,
    425        )
    426    }
    427
    428    fn finish_download(
    429        &mut self,
    430        pkg: PackageId,
    431        checksum: &str,
    432        data: &[u8],
    433    ) -> CargoResult<File> {
    434        download::finish_download(
    435            &self.cache_path,
    436            &self.gctx,
    437            self.name.clone(),
    438            pkg,
    439            checksum,
    440            data,
    441        )
    442    }
    443
    444    fn is_crate_downloaded(&self, pkg: PackageId) -> bool {
    445        download::is_crate_downloaded(&self.cache_path, &self.gctx, pkg)
    446    }
    447}
    448
    449/// Implemented to just be sure to drop `tree` field before our other fields.
    450/// See SAFETY inside [`RemoteRegistry::tree()`] for more.
    451impl<'gctx> Drop for RemoteRegistry<'gctx> {
    452    fn drop(&mut self) {
    453        self.tree.borrow_mut().take();
    454    }
    455}