# Copyright 2009-2010 Gregory P. Ward # Copyright 2009-2010 Intelerad Medical Systems Incorporated # Copyright 2010-2011 Fog Creek Software # Copyright 2010-2011 Unity Technologies # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. '''base class for store implementations and store-related utility code''' import re from mercurial import util, node, hg from mercurial.i18n import _ import lfutil class StoreError(Exception): '''Raised when there is a problem getting files from or putting files to a central store.''' def __init__(self, filename, hash, url, detail): self.filename = filename self.hash = hash self.url = url self.detail = detail def longmessage(self): return (_("error getting id %s from url %s for file %s: %s\n") % (self.hash, self.url, self.filename, self.detail)) def __str__(self): return "%s: %s" % (self.url, self.detail) class basestore(object): def __init__(self, ui, repo, url): self.ui = ui self.repo = repo self.url = url def put(self, source, hash): '''Put source file into the store so it can be retrieved by hash.''' raise NotImplementedError('abstract method') def exists(self, hashes): '''Check to see if the store contains the given hashes. Given an iterable of hashes it returns a mapping from hash to bool.''' raise NotImplementedError('abstract method') def get(self, files): '''Get the specified largefiles from the store and write to local files under repo.root. files is a list of (filename, hash) tuples. Return (success, missing), lists of files successfully downloaded and those not found in the store. success is a list of (filename, hash) tuples; missing is a list of filenames that we could not get. (The detailed error message will already have been presented to the user, so missing is just supplied as a summary.)''' success = [] missing = [] ui = self.ui util.makedirs(lfutil.storepath(self.repo, '')) at = 0 available = self.exists(set(hash for (_filename, hash) in files)) for filename, hash in files: ui.progress(_('getting largefiles'), at, unit='lfile', total=len(files)) at += 1 ui.note(_('getting %s:%s\n') % (filename, hash)) if not available.get(hash): ui.warn(_('%s: largefile %s not available from %s\n') % (filename, hash, self.url)) missing.append(filename) continue storefilename = lfutil.storepath(self.repo, hash) tmpfile = util.atomictempfile(storefilename + '.tmp', createmode=self.repo.store.createmode) try: hhash = self._getfile(tmpfile, filename, hash) except StoreError, err: ui.warn(err.longmessage()) hhash = "" tmpfile.close() if hhash != hash: if hhash != "": ui.warn(_('%s: data corruption (expected %s, got %s)\n') % (filename, hash, hhash)) util.unlink(storefilename + '.tmp') missing.append(filename) continue util.rename(storefilename + '.tmp', storefilename) lfutil.linktousercache(self.repo, hash) success.append((filename, hhash)) ui.progress(_('getting largefiles'), None) return (success, missing) def verify(self, revs, contents=False): '''Verify the existence (and, optionally, contents) of every big file revision referenced by every changeset in revs. Return 0 if all is well, non-zero on any errors.''' failed = False self.ui.status(_('searching %d changesets for largefiles\n') % len(revs)) verified = set() # set of (filename, filenode) tuples for rev in revs: cctx = self.repo[rev] cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) for standin in cctx: if self._verifyfile(cctx, cset, contents, standin, verified): failed = True numrevs = len(verified) numlfiles = len(set([fname for (fname, fnode) in verified])) if contents: self.ui.status( _('verified contents of %d revisions of %d largefiles\n') % (numrevs, numlfiles)) else: self.ui.status( _('verified existence of %d revisions of %d largefiles\n') % (numrevs, numlfiles)) return int(failed) def _getfile(self, tmpfile, filename, hash): '''Fetch one revision of one file from the store and write it to tmpfile. Compute the hash of the file on-the-fly as it downloads and return the hash. Close tmpfile. Raise StoreError if unable to download the file (e.g. it does not exist in the store).''' raise NotImplementedError('abstract method') def _verifyfile(self, cctx, cset, contents, standin, verified): '''Perform the actual verification of a file in the store. 'cset' is only used in warnings. 'contents' controls verification of content hash. 'standin' is the standin path of the largefile to verify. 'verified' is maintained as a set of already verified files. Returns _true_ if it is a standin and any problems are found! ''' raise NotImplementedError('abstract method') import localstore, wirestore _storeprovider = { 'file': [localstore.localstore], 'http': [wirestore.wirestore], 'https': [wirestore.wirestore], 'ssh': [wirestore.wirestore], } _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') # During clone this function is passed the src's ui object # but it needs the dest's ui object so it can read out of # the config file. Use repo.ui instead. def _openstore(repo, remote=None, put=False): ui = repo.ui if not remote: lfpullsource = getattr(repo, 'lfpullsource', None) if lfpullsource: path = ui.expandpath(lfpullsource) else: path = ui.expandpath('default-push', 'default') # ui.expandpath() leaves 'default-push' and 'default' alone if # they cannot be expanded: fallback to the empty string, # meaning the current directory. if path == 'default-push' or path == 'default': path = '' remote = repo else: path, _branches = hg.parseurl(path) remote = hg.peer(repo, {}, path) # The path could be a scheme so use Mercurial's normal functionality # to resolve the scheme to a repository and use its path path = util.safehasattr(remote, 'url') and remote.url() or remote.path match = _scheme_re.match(path) if not match: # regular filesystem path scheme = 'file' else: scheme = match.group(1) try: storeproviders = _storeprovider[scheme] except KeyError: raise util.Abort(_('unsupported URL scheme %r') % scheme) for classobj in storeproviders: try: return classobj(ui, repo, remote) except lfutil.storeprotonotcapable: pass raise util.Abort(_('%s does not appear to be a largefile store') % path)