1
# Copyright (C) 2011, 2012 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""bzr-upload command implementations."""
19
from __future__ import absolute_import
28
lazy_import.lazy_import(globals(), """
43
from ...sixish import (
47
auto_option = config.Option(
48
'upload_auto', default=False, from_unicode=config.bool_from_store,
50
Whether upload should occur when the tip of the branch changes.
52
auto_quiet_option = config.Option(
53
'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
55
Whether upload should occur quietly.
57
location_option = config.Option(
58
'upload_location', default=None,
60
The url to upload the working tree to.
62
revid_location_option = config.Option(
63
'upload_revid_location', default=u'.bzr-upload.revid',
65
The relative path to be used to store the uploaded revid.
67
The only bzr-related info uploaded with the working tree is the corresponding
68
revision id. The uploaded working tree is not linked to any other bzr data.
70
If the layout of your remote server is such that you can't write in the
71
root directory but only in the directories inside that root, you will need
72
to use the 'upload_revid_location' configuration variable to specify the
73
relative path to be used. That configuration variable can be specified in
74
locations.conf or branch.conf.
76
For example, given the following layout:
82
you may have write access in 'private' and 'public' but in 'Project'
83
itself. In that case, you can add the following in your locations.conf or
86
upload_revid_location = private/.bzr-upload.revid
90
# FIXME: Add more tests around invalid paths or relative paths that doesn't
91
# exist on remote (if only to get proper error messages) for
92
# 'upload_revid_location'
95
class BzrUploader(object):
97
def __init__(self, branch, to_transport, outf, tree, rev_id,
100
self.to_transport = to_transport
105
self._pending_deletions = []
106
self._pending_renames = []
107
self._uploaded_revid = None
110
def _up_stat(self, relpath):
111
return self.to_transport.stat(urlutils.escape(relpath))
113
def _up_rename(self, old_path, new_path):
114
return self.to_transport.rename(urlutils.escape(old_path),
115
urlutils.escape(new_path))
117
def _up_delete(self, relpath):
118
return self.to_transport.delete(urlutils.escape(relpath))
120
def _up_delete_tree(self, relpath):
121
return self.to_transport.delete_tree(urlutils.escape(relpath))
123
def _up_mkdir(self, relpath, mode):
124
return self.to_transport.mkdir(urlutils.escape(relpath), mode)
126
def _up_rmdir(self, relpath):
127
return self.to_transport.rmdir(urlutils.escape(relpath))
129
def _up_put_bytes(self, relpath, bytes, mode):
130
self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
132
def _up_get_bytes(self, relpath):
133
return self.to_transport.get_bytes(urlutils.escape(relpath))
135
def set_uploaded_revid(self, rev_id):
136
# XXX: Add tests for concurrent updates, etc.
137
revid_path = self.branch.get_config_stack().get('upload_revid_location')
138
self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
139
self._uploaded_revid = rev_id
141
def get_uploaded_revid(self):
142
if self._uploaded_revid is None:
143
revid_path = self.branch.get_config_stack(
144
).get('upload_revid_location')
146
self._uploaded_revid = self._up_get_bytes(revid_path)
147
except errors.NoSuchFile:
148
# We have not uploaded to here.
149
self._uploaded_revid = revision.NULL_REVISION
150
return self._uploaded_revid
152
def _get_ignored(self):
153
if self._ignored is None:
155
ignore_file_path = '.bzrignore-upload'
156
ignore_file_id = self.tree.path2id(ignore_file_path)
157
ignore_file = self.tree.get_file(ignore_file_id,
159
ignored_patterns = ignores.parse_ignore_file(ignore_file)
160
except errors.NoSuchId:
161
ignored_patterns = []
162
self._ignored = globbing.Globster(ignored_patterns)
165
def is_ignored(self, relpath):
166
glob = self._get_ignored()
167
ignored = glob.match(relpath)
170
# We still need to check that all parents are not ignored
171
dir = os.path.dirname(relpath)
172
while dir and not ignored:
173
ignored = glob.match(dir)
175
dir = os.path.dirname(dir)
178
def upload_file(self, relpath, id, mode=None):
180
if self.tree.is_executable(id):
185
self.outf.write('Uploading %s\n' % relpath)
186
self._up_put_bytes(relpath, self.tree.get_file_text(id), mode)
188
def upload_file_robustly(self, relpath, id, mode=None):
189
"""Upload a file, clearing the way on the remote side.
191
When doing a full upload, it may happen that a directory exists where
192
we want to put our file.
195
st = self._up_stat(relpath)
196
if stat.S_ISDIR(st.st_mode):
197
# A simple rmdir may not be enough
199
self.outf.write('Clearing %s/%s\n' % (
200
self.to_transport.external_url(), relpath))
201
self._up_delete_tree(relpath)
202
except errors.PathError:
204
self.upload_file(relpath, id, mode)
206
def make_remote_dir(self, relpath, mode=None):
209
self._up_mkdir(relpath, mode)
211
def make_remote_dir_robustly(self, relpath, mode=None):
212
"""Create a remote directory, clearing the way on the remote side.
214
When doing a full upload, it may happen that a file exists where we
215
want to create our directory.
218
st = self._up_stat(relpath)
219
if not stat.S_ISDIR(st.st_mode):
221
self.outf.write('Deleting %s/%s\n' % (
222
self.to_transport.external_url(), relpath))
223
self._up_delete(relpath)
225
# Ok the remote dir already exists, nothing to do
227
except errors.PathError:
229
self.make_remote_dir(relpath, mode)
231
def delete_remote_file(self, relpath):
233
self.outf.write('Deleting %s\n' % relpath)
234
self._up_delete(relpath)
236
def delete_remote_dir(self, relpath):
238
self.outf.write('Deleting %s\n' % relpath)
239
self._up_rmdir(relpath)
240
# XXX: Add a test where a subdir is ignored but we still want to
241
# delete the dir -- vila 100106
243
def delete_remote_dir_maybe(self, relpath):
244
"""Try to delete relpath, keeping failures to retry later."""
246
self._up_rmdir(relpath)
247
# any kind of PathError would be OK, though we normally expect
249
except errors.PathError:
250
self._pending_deletions.append(relpath)
252
def finish_deletions(self):
253
if self._pending_deletions:
254
# Process the previously failed deletions in reverse order to
255
# delete children before parents
256
for relpath in reversed(self._pending_deletions):
257
self._up_rmdir(relpath)
258
# The following shouldn't be needed since we use it once per
259
# upload, but better safe than sorry ;-)
260
self._pending_deletions = []
262
def rename_remote(self, old_relpath, new_relpath):
263
"""Rename a remote file or directory taking care of collisions.
265
To avoid collisions during bulk renames, each renamed target is
266
temporarily assigned a unique name. When all renames have been done,
267
each target get its proper name.
269
# We generate a sufficiently random name to *assume* that
270
# no collisions will occur and don't worry about it (nor
276
stamp = '.tmp.%.9f.%d.%d' % (time.time(),
278
random.randint(0,0x7FFFFFFF))
280
self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
281
self._up_rename(old_relpath, stamp)
282
self._pending_renames.append((stamp, new_relpath))
284
def finish_renames(self):
285
for (stamp, new_path) in self._pending_renames:
286
self._up_rename(stamp, new_path)
287
# The following shouldn't be needed since we use it once per upload,
288
# but better safe than sorry ;-)
289
self._pending_renames = []
291
def upload_full_tree(self):
292
self.to_transport.ensure_base() # XXX: Handle errors (add
293
# --create-prefix option ?)
294
with self.tree.lock_read():
295
for relpath, ie in self.tree.iter_entries_by_dir():
296
if relpath in ('', '.bzrignore', '.bzrignore-upload'):
298
# .bzrignore and .bzrignore-upload have no meaning outside
299
# a working tree so do not upload them
301
if self.is_ignored(relpath):
303
self.outf.write('Ignoring %s\n' % relpath)
305
if ie.kind == 'file':
306
self.upload_file_robustly(relpath, ie.file_id)
307
elif ie.kind == 'directory':
308
self.make_remote_dir_robustly(relpath)
309
elif ie.kind == 'symlink':
311
target = self.tree.path_content_summary(relpath)[3]
312
self.outf.write('Not uploading symlink %s -> %s\n'
315
raise NotImplementedError
316
self.set_uploaded_revid(self.rev_id)
318
def upload_tree(self):
319
# If we can't find the revid file on the remote location, upload the
321
rev_id = self.get_uploaded_revid()
323
if rev_id == revision.NULL_REVISION:
325
self.outf.write('No uploaded revision id found,'
326
' switching to full upload\n')
327
self.upload_full_tree()
331
# Check if the revision hasn't already been uploaded
332
if rev_id == self.rev_id:
334
self.outf.write('Remote location already up to date\n')
336
from_tree = self.branch.repository.revision_tree(rev_id)
337
self.to_transport.ensure_base() # XXX: Handle errors (add
338
# --create-prefix option ?)
339
changes = self.tree.changes_from(from_tree)
340
with self.tree.lock_read():
341
for (path, id, kind) in changes.removed:
342
if self.is_ignored(path):
344
self.outf.write('Ignoring %s\n' % path)
347
self.delete_remote_file(path)
348
elif kind is 'directory':
349
self.delete_remote_dir_maybe(path)
350
elif kind == 'symlink':
352
target = self.tree.path_content_summary(path)[3]
353
self.outf.write('Not deleting remote symlink %s -> %s\n'
356
raise NotImplementedError
358
for (old_path, new_path, id, kind,
359
content_change, exec_change) in changes.renamed:
360
if self.is_ignored(old_path) and self.is_ignored(new_path):
362
self.outf.write('Ignoring %s\n' % old_path)
363
self.outf.write('Ignoring %s\n' % new_path)
366
# We update the old_path content because renames and
367
# deletions are differed.
368
self.upload_file(old_path, id)
369
if kind == 'symlink':
371
self.outf.write('Not renaming remote symlink %s to %s\n'
372
% (old_path, new_path))
374
self.rename_remote(old_path, new_path)
375
self.finish_renames()
376
self.finish_deletions()
378
for (path, id, old_kind, new_kind) in changes.kind_changed:
379
if self.is_ignored(path):
381
self.outf.write('Ignoring %s\n' % path)
383
if old_kind == 'file':
384
self.delete_remote_file(path)
385
elif old_kind == 'directory':
386
self.delete_remote_dir(path)
388
raise NotImplementedError
390
if new_kind == 'file':
391
self.upload_file(path, id)
392
elif new_kind is 'directory':
393
self.make_remote_dir(path)
395
raise NotImplementedError
397
for (path, id, kind) in changes.added:
398
if self.is_ignored(path):
400
self.outf.write('Ignoring %s\n' % path)
403
self.upload_file(path, id)
404
elif kind == 'directory':
405
self.make_remote_dir(path)
406
elif kind == 'symlink':
408
target = self.tree.path_content_summary(path)[3]
409
self.outf.write('Not uploading symlink %s -> %s\n'
412
raise NotImplementedError
414
# XXX: Add a test for exec_change
416
content_change, exec_change) in changes.modified:
417
if self.is_ignored(path):
419
self.outf.write('Ignoring %s\n' % path)
422
self.upload_file(path, id)
424
raise NotImplementedError
426
self.set_uploaded_revid(self.rev_id)
429
class CannotUploadToWorkingTree(errors.BzrCommandError):
431
_fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
434
class DivergedUploadedTree(errors.BzrCommandError):
436
_fmt = ("Your branch (%(revid)s)"
437
" and the uploaded tree (%(uploaded_revid)s) have diverged: ")
440
class cmd_upload(commands.Command):
441
"""Upload a working tree, as a whole or incrementally.
443
If no destination is specified use the last one used.
444
If no revision is specified upload the changes since the last upload.
446
Changes include files added, renamed, modified or removed.
448
_see_also = ['plugins/upload']
449
takes_args = ['location?']
454
option.Option('full', 'Upload the full working tree.'),
455
option.Option('quiet', 'Do not output what is being done.',
457
option.Option('directory',
458
help='Branch to upload from, '
459
'rather than the one containing the working directory.',
463
option.Option('auto',
464
'Trigger an upload from this branch whenever the tip '
468
def run(self, location=None, full=False, revision=None, remember=None,
469
directory=None, quiet=False, auto=None, overwrite=False
471
if directory is None:
475
relpath) = controldir.ControlDir.open_containing_tree_or_branch(
486
changes = wt.changes_from(wt.basis_tree())
488
if revision is None and changes.has_changed():
489
raise errors.UncommittedChanges(wt)
491
conf = branch.get_config_stack()
493
stored_loc = conf.get('upload_location')
494
if stored_loc is None:
495
raise errors.BzrCommandError(
496
'No upload location known or specified.')
498
# FIXME: Not currently tested
499
display_url = urlutils.unescape_for_display(stored_loc,
501
self.outf.write("Using saved location: %s\n" % display_url)
502
location = stored_loc
504
to_transport = transport.get_transport(location)
506
# Check that we are not uploading to a existing working tree.
508
to_bzr_dir = controldir.ControlDir.open_from_transport(
510
has_wt = to_bzr_dir.has_workingtree()
511
except errors.NotBranchError:
513
except errors.NotLocalUrl:
514
# The exception raised is a bit weird... but that's life.
518
raise CannotUploadToWorkingTree(url=location)
520
rev_id = branch.last_revision()
522
if len(revision) != 1:
523
raise errors.BzrCommandError(
524
'bzr upload --revision takes exactly 1 argument')
525
rev_id = revision[0].in_history(branch).rev_id
527
tree = branch.repository.revision_tree(rev_id)
529
uploader = BzrUploader(branch, to_transport, self.outf, tree,
533
prev_uploaded_rev_id = uploader.get_uploaded_revid()
534
graph = branch.repository.get_graph()
535
if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
536
raise DivergedUploadedTree(
537
revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
539
uploader.upload_full_tree()
541
uploader.upload_tree()
545
# We uploaded successfully, remember it
548
upload_location = conf.get('upload_location')
549
if upload_location is None or remember:
550
conf.set('upload_location',
551
urlutils.unescape(to_transport.base))
553
conf.set('upload_auto', auto)
added
removed
breezy/plugins/upload/cmds.py
