1
# Copyright (C) 2011, 2012 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""bzr-upload command implementations."""
19
from __future__ import absolute_import
29
lazy_import.lazy_import(globals(), """
42
from ...sixish import (
46
auto_option = config.Option(
47
'upload_auto', default=False, from_unicode=config.bool_from_store,
49
Whether upload should occur when the tip of the branch changes.
51
auto_quiet_option = config.Option(
52
'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
54
Whether upload should occur quietly.
56
location_option = config.Option(
57
'upload_location', default=None,
59
The url to upload the working tree to.
61
revid_location_option = config.Option(
62
'upload_revid_location', default=u'.bzr-upload.revid',
64
The relative path to be used to store the uploaded revid.
66
The only bzr-related info uploaded with the working tree is the corresponding
67
revision id. The uploaded working tree is not linked to any other bzr data.
69
If the layout of your remote server is such that you can't write in the
70
root directory but only in the directories inside that root, you will need
71
to use the 'upload_revid_location' configuration variable to specify the
72
relative path to be used. That configuration variable can be specified in
73
locations.conf or branch.conf.
75
For example, given the following layout:
81
you may have write access in 'private' and 'public' but in 'Project'
82
itself. In that case, you can add the following in your locations.conf or
85
upload_revid_location = private/.bzr-upload.revid
89
# FIXME: Add more tests around invalid paths or relative paths that doesn't
90
# exist on remote (if only to get proper error messages) for
91
# 'upload_revid_location'
94
class BzrUploader(object):
96
def __init__(self, branch, to_transport, outf, tree, rev_id,
99
self.to_transport = to_transport
104
self._pending_deletions = []
105
self._pending_renames = []
106
self._uploaded_revid = None
109
def _up_stat(self, relpath):
110
return self.to_transport.stat(urlutils.escape(relpath))
112
def _up_rename(self, old_path, new_path):
113
return self.to_transport.rename(urlutils.escape(old_path),
114
urlutils.escape(new_path))
116
def _up_delete(self, relpath):
117
return self.to_transport.delete(urlutils.escape(relpath))
119
def _up_delete_tree(self, relpath):
120
return self.to_transport.delete_tree(urlutils.escape(relpath))
122
def _up_mkdir(self, relpath, mode):
123
return self.to_transport.mkdir(urlutils.escape(relpath), mode)
125
def _up_rmdir(self, relpath):
126
return self.to_transport.rmdir(urlutils.escape(relpath))
128
def _up_put_bytes(self, relpath, bytes, mode):
129
self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
131
def _up_get_bytes(self, relpath):
132
return self.to_transport.get_bytes(urlutils.escape(relpath))
134
def set_uploaded_revid(self, rev_id):
135
# XXX: Add tests for concurrent updates, etc.
136
revid_path = self.branch.get_config_stack().get('upload_revid_location')
137
self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
138
self._uploaded_revid = rev_id
140
def get_uploaded_revid(self):
141
if self._uploaded_revid is None:
142
revid_path = self.branch.get_config_stack(
143
).get('upload_revid_location')
145
self._uploaded_revid = self._up_get_bytes(revid_path)
146
except errors.NoSuchFile:
147
# We have not uploaded to here.
148
self._uploaded_revid = revision.NULL_REVISION
149
return self._uploaded_revid
151
def _get_ignored(self):
152
if self._ignored is None:
154
ignore_file_path = '.bzrignore-upload'
155
ignore_file = self.tree.get_file(ignore_file_path)
156
except errors.NoSuchFile:
157
ignored_patterns = []
159
ignored_patterns = ignores.parse_ignore_file(ignore_file)
160
self._ignored = globbing.Globster(ignored_patterns)
163
def is_ignored(self, relpath):
164
glob = self._get_ignored()
165
ignored = glob.match(relpath)
168
# We still need to check that all parents are not ignored
169
dir = os.path.dirname(relpath)
170
while dir and not ignored:
171
ignored = glob.match(dir)
173
dir = os.path.dirname(dir)
176
def upload_file(self, old_relpath, new_relpath, mode=None):
178
if self.tree.is_executable(new_relpath):
183
self.outf.write('Uploading %s\n' % old_relpath)
185
old_relpath, self.tree.get_file_text(new_relpath), mode)
187
def _force_clear(self, relpath):
189
st = self._up_stat(relpath)
190
if stat.S_ISDIR(st.st_mode):
191
# A simple rmdir may not be enough
193
self.outf.write('Clearing %s/%s\n' % (
194
self.to_transport.external_url(), relpath))
195
self._up_delete_tree(relpath)
196
elif stat.S_ISLNK(st.st_mode):
198
self.outf.write('Clearing %s/%s\n' % (
199
self.to_transport.external_url(), relpath))
200
self._up_delete(relpath)
201
except errors.PathError:
204
def upload_file_robustly(self, relpath, mode=None):
205
"""Upload a file, clearing the way on the remote side.
207
When doing a full upload, it may happen that a directory exists where
208
we want to put our file.
210
self._force_clear(relpath)
211
self.upload_file(relpath, relpath, mode)
213
def upload_symlink(self, relpath, target):
214
self.to_transport.symlink(target, relpath)
216
def upload_symlink_robustly(self, relpath, target):
217
"""Handle uploading symlinks.
219
self._force_clear(relpath)
220
# Target might not be there at this time; dummy file should be
221
# overwritten at some point, possibly by another upload.
222
target = osutils.normpath(osutils.pathjoin(
223
osutils.dirname(relpath),
226
self.upload_symlink(relpath, target)
228
def make_remote_dir(self, relpath, mode=None):
231
self._up_mkdir(relpath, mode)
233
def make_remote_dir_robustly(self, relpath, mode=None):
234
"""Create a remote directory, clearing the way on the remote side.
236
When doing a full upload, it may happen that a file exists where we
237
want to create our directory.
240
st = self._up_stat(relpath)
241
if not stat.S_ISDIR(st.st_mode):
243
self.outf.write('Deleting %s/%s\n' % (
244
self.to_transport.external_url(), relpath))
245
self._up_delete(relpath)
247
# Ok the remote dir already exists, nothing to do
249
except errors.PathError:
251
self.make_remote_dir(relpath, mode)
253
def delete_remote_file(self, relpath):
255
self.outf.write('Deleting %s\n' % relpath)
256
self._up_delete(relpath)
258
def delete_remote_dir(self, relpath):
260
self.outf.write('Deleting %s\n' % relpath)
261
self._up_rmdir(relpath)
262
# XXX: Add a test where a subdir is ignored but we still want to
263
# delete the dir -- vila 100106
265
def delete_remote_dir_maybe(self, relpath):
266
"""Try to delete relpath, keeping failures to retry later."""
268
self._up_rmdir(relpath)
269
# any kind of PathError would be OK, though we normally expect
271
except errors.PathError:
272
self._pending_deletions.append(relpath)
274
def finish_deletions(self):
275
if self._pending_deletions:
276
# Process the previously failed deletions in reverse order to
277
# delete children before parents
278
for relpath in reversed(self._pending_deletions):
279
self._up_rmdir(relpath)
280
# The following shouldn't be needed since we use it once per
281
# upload, but better safe than sorry ;-)
282
self._pending_deletions = []
284
def rename_remote(self, old_relpath, new_relpath):
285
"""Rename a remote file or directory taking care of collisions.
287
To avoid collisions during bulk renames, each renamed target is
288
temporarily assigned a unique name. When all renames have been done,
289
each target get its proper name.
291
# We generate a sufficiently random name to *assume* that
292
# no collisions will occur and don't worry about it (nor
298
stamp = '.tmp.%.9f.%d.%d' % (time.time(),
300
random.randint(0, 0x7FFFFFFF))
302
self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
303
self._up_rename(old_relpath, stamp)
304
self._pending_renames.append((stamp, new_relpath))
306
def finish_renames(self):
307
for (stamp, new_path) in self._pending_renames:
308
self._up_rename(stamp, new_path)
309
# The following shouldn't be needed since we use it once per upload,
310
# but better safe than sorry ;-)
311
self._pending_renames = []
313
def upload_full_tree(self):
314
self.to_transport.ensure_base() # XXX: Handle errors (add
315
# --create-prefix option ?)
316
with self.tree.lock_read():
317
for relpath, ie in self.tree.iter_entries_by_dir():
318
if relpath in ('', '.bzrignore', '.bzrignore-upload'):
320
# .bzrignore and .bzrignore-upload have no meaning outside
321
# a working tree so do not upload them
323
if self.is_ignored(relpath):
325
self.outf.write('Ignoring %s\n' % relpath)
327
if ie.kind == 'file':
328
self.upload_file_robustly(relpath)
329
elif ie.kind == 'symlink':
331
self.upload_symlink_robustly(
332
relpath, ie.symlink_target)
333
except errors.TransportNotPossible:
335
target = self.tree.path_content_summary(relpath)[3]
336
self.outf.write('Not uploading symlink %s -> %s\n'
338
elif ie.kind == 'directory':
339
self.make_remote_dir_robustly(relpath)
341
raise NotImplementedError
342
self.set_uploaded_revid(self.rev_id)
344
def upload_tree(self):
345
# If we can't find the revid file on the remote location, upload the
347
rev_id = self.get_uploaded_revid()
349
if rev_id == revision.NULL_REVISION:
351
self.outf.write('No uploaded revision id found,'
352
' switching to full upload\n')
353
self.upload_full_tree()
357
# Check if the revision hasn't already been uploaded
358
if rev_id == self.rev_id:
360
self.outf.write('Remote location already up to date\n')
362
from_tree = self.branch.repository.revision_tree(rev_id)
363
self.to_transport.ensure_base() # XXX: Handle errors (add
364
# --create-prefix option ?)
365
changes = self.tree.changes_from(from_tree)
366
with self.tree.lock_read():
367
for change in changes.removed:
368
if self.is_ignored(change.path[0]):
370
self.outf.write('Ignoring %s\n' % change.path[0])
372
if change.kind[0] == 'file':
373
self.delete_remote_file(change.path[0])
374
elif change.kind[0] == 'directory':
375
self.delete_remote_dir_maybe(change.path[0])
376
elif change.kind[0] == 'symlink':
377
self.delete_remote_file(change.path[0])
379
raise NotImplementedError
381
for change in changes.renamed:
382
if self.is_ignored(change.path[0]) and self.is_ignored(change.path[1]):
384
self.outf.write('Ignoring %s\n' % change.path[0])
385
self.outf.write('Ignoring %s\n' % change.path[1])
387
if change.changed_content:
388
# We update the change.path[0] content because renames and
389
# deletions are differed.
390
self.upload_file(change.path[0], change.path[1])
391
self.rename_remote(change.path[0], change.path[1])
392
self.finish_renames()
393
self.finish_deletions()
395
for change in changes.kind_changed:
396
if self.is_ignored(change.path[1]):
398
self.outf.write('Ignoring %s\n' % change.path[1])
400
if change.kind[0] in ('file', 'symlink'):
401
self.delete_remote_file(change.path[0])
402
elif change.kind[0] == 'directory':
403
self.delete_remote_dir(change.path[0])
405
raise NotImplementedError
407
if change.kind[1] == 'file':
408
self.upload_file(change.path[1], change.path[1])
409
elif change.kind[1] == 'symlink':
410
target = self.tree.get_symlink_target(change.path[1])
411
self.upload_symlink(change.path[1], target)
412
elif change.kind[1] == 'directory':
413
self.make_remote_dir(change.path[1])
415
raise NotImplementedError
417
for change in changes.added + changes.copied:
418
if self.is_ignored(change.path[1]):
420
self.outf.write('Ignoring %s\n' % change.path[1])
422
if change.kind[1] == 'file':
423
self.upload_file(change.path[1], change.path[1])
424
elif change.kind[1] == 'directory':
425
self.make_remote_dir(change.path[1])
426
elif change.kind[1] == 'symlink':
427
target = self.tree.get_symlink_target(change.path[1])
429
self.upload_symlink(change.path[1], target)
430
except errors.TransportNotPossible:
432
self.outf.write('Not uploading symlink %s -> %s\n'
433
% (change.path[1], target))
435
raise NotImplementedError
437
# XXX: Add a test for exec_change
438
for change in changes.modified:
439
if self.is_ignored(change.path[1]):
441
self.outf.write('Ignoring %s\n' % change.path[1])
443
if change.kind[1] == 'file':
444
self.upload_file(change.path[1], change.path[1])
445
elif change.kind[1] == 'symlink':
446
target = self.tree.get_symlink_target(change.path[1])
447
self.upload_symlink(change.path[1], target)
449
raise NotImplementedError
451
self.set_uploaded_revid(self.rev_id)
454
class CannotUploadToWorkingTree(errors.BzrCommandError):
456
_fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
459
class DivergedUploadedTree(errors.BzrCommandError):
461
_fmt = ("Your branch (%(revid)s)"
462
" and the uploaded tree (%(uploaded_revid)s) have diverged: ")
465
class cmd_upload(commands.Command):
466
"""Upload a working tree, as a whole or incrementally.
468
If no destination is specified use the last one used.
469
If no revision is specified upload the changes since the last upload.
471
Changes include files added, renamed, modified or removed.
473
_see_also = ['plugins/upload']
474
takes_args = ['location?']
479
option.Option('full', 'Upload the full working tree.'),
480
option.Option('quiet', 'Do not output what is being done.',
482
option.Option('directory',
483
help='Branch to upload from, '
484
'rather than the one containing the working directory.',
488
option.Option('auto',
489
'Trigger an upload from this branch whenever the tip '
493
def run(self, location=None, full=False, revision=None, remember=None,
494
directory=None, quiet=False, auto=None, overwrite=False
496
if directory is None:
500
relpath) = controldir.ControlDir.open_containing_tree_or_branch(
507
with locked.lock_read():
509
changes = wt.changes_from(wt.basis_tree())
511
if revision is None and changes.has_changed():
512
raise errors.UncommittedChanges(wt)
514
conf = branch.get_config_stack()
516
stored_loc = conf.get('upload_location')
517
if stored_loc is None:
518
raise errors.BzrCommandError(
519
'No upload location known or specified.')
521
# FIXME: Not currently tested
522
display_url = urlutils.unescape_for_display(stored_loc,
524
self.outf.write("Using saved location: %s\n" % display_url)
525
location = stored_loc
527
to_transport = transport.get_transport(location)
529
# Check that we are not uploading to a existing working tree.
531
to_bzr_dir = controldir.ControlDir.open_from_transport(
533
has_wt = to_bzr_dir.has_workingtree()
534
except errors.NotBranchError:
536
except errors.NotLocalUrl:
537
# The exception raised is a bit weird... but that's life.
541
raise CannotUploadToWorkingTree(url=location)
543
rev_id = branch.last_revision()
545
if len(revision) != 1:
546
raise errors.BzrCommandError(
547
'bzr upload --revision takes exactly 1 argument')
548
rev_id = revision[0].in_history(branch).rev_id
550
tree = branch.repository.revision_tree(rev_id)
552
uploader = BzrUploader(branch, to_transport, self.outf, tree,
556
prev_uploaded_rev_id = uploader.get_uploaded_revid()
557
graph = branch.repository.get_graph()
558
if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
559
raise DivergedUploadedTree(
560
revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
562
uploader.upload_full_tree()
564
uploader.upload_tree()
566
# We uploaded successfully, remember it
567
with branch.lock_write():
568
upload_location = conf.get('upload_location')
569
if upload_location is None or remember:
570
conf.set('upload_location',
571
urlutils.unescape(to_transport.base))
573
conf.set('upload_auto', auto)
added
removed
breezy/plugins/upload/cmds.py
